ydb/contrib/python/Twisted/py2/twisted/trial/_dist/disttrial.py

# -*- test-case-name: twisted.trial._dist.test.test_disttrial -*-
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.

"""
This module contains the trial distributed runner, the management class
responsible for coordinating all of trial's behavior at the highest level.

@since: 12.3
"""

import os
import sys

from twisted.python.filepath import FilePath
from twisted.python.modules import theSystemPath
from twisted.internet.defer import DeferredList
from twisted.internet.task import cooperate

from twisted.trial.util import _unusedTestDirectory
from twisted.trial._asyncrunner import _iterateTests
from twisted.trial._dist.worker import LocalWorker, LocalWorkerAMP
from twisted.trial._dist.distreporter import DistReporter
from twisted.trial.reporter import UncleanWarningsReporterWrapper
from twisted.trial._dist import _WORKER_AMP_STDIN, _WORKER_AMP_STDOUT



class DistTrialRunner(object):
    """
    A specialized runner for distributed trial. The runner launches a number of
    local worker processes which will run tests.

    @ivar _workerNumber: the number of workers to be spawned.
    @type _workerNumber: C{int}

    @ivar _stream: stream which the reporter will use.

    @ivar _reporterFactory: the reporter class to be used.
    """
    _distReporterFactory = DistReporter

    def _makeResult(self):
        """
        Make reporter factory, and wrap it with a L{DistReporter}.
        """
        reporter = self._reporterFactory(self._stream, self._tbformat,
                                         realtime=self._rterrors)
        if self._uncleanWarnings:
            reporter = UncleanWarningsReporterWrapper(reporter)
        return self._distReporterFactory(reporter)


    def __init__(self, reporterFactory, workerNumber, workerArguments,
                 stream=None,
                 tracebackFormat='default',
                 realTimeErrors=False,
                 uncleanWarnings=False,
                 logfile='test.log',
                 workingDirectory='_trial_temp'):
        self._workerNumber = workerNumber
        self._workerArguments = workerArguments
        self._reporterFactory = reporterFactory
        if stream is None:
            stream = sys.stdout
        self._stream = stream
        self._tbformat = tracebackFormat
        self._rterrors = realTimeErrors
        self._uncleanWarnings = uncleanWarnings
        self._result = None
        self._workingDirectory = workingDirectory
        self._logFile = logfile
        self._logFileObserver = None
        self._logFileObject = None
        self._logWarnings = False


    def writeResults(self, result):
        """
        Write test run final outcome to result.

        @param result: A C{TestResult} which will print errors and the summary.
        """
        result.done()


    def createLocalWorkers(self, protocols, workingDirectory):
        """
        Create local worker protocol instances and return them.

        @param protocols: An iterable of L{LocalWorkerAMP} instances.

        @param workingDirectory: The base path in which we should run the
            workers.
        @type workingDirectory: C{str}

        @return: A list of C{quantity} C{LocalWorker} instances.
        """
        return [LocalWorker(protocol,
                            os.path.join(workingDirectory, str(x)),
                            self._logFile)
                for x, protocol in enumerate(protocols)]


    def launchWorkerProcesses(self, spawner, protocols, arguments):
        """
        Spawn processes from a list of process protocols.

        @param spawner: A C{IReactorProcess.spawnProcess} implementation.

        @param protocols: An iterable of C{ProcessProtocol} instances.

        @param arguments: Extra arguments passed to the processes.
        """
        workertrialPath = theSystemPath[
            'twisted.trial._dist.workertrial'].filePath.path
        childFDs = {0: 'w', 1: 'r', 2: 'r', _WORKER_AMP_STDIN: 'w',
                    _WORKER_AMP_STDOUT: 'r'}
        environ = os.environ.copy()
        # Add an environment variable containing the raw sys.path, to be used by
        # subprocesses to make sure it's identical to the parent. See
        # workertrial._setupPath.
        environ['TRIAL_PYTHONPATH'] = os.pathsep.join(sys.path)
        for worker in protocols:
            args = [sys.executable, workertrialPath]
            args.extend(arguments)
            spawner(worker, sys.executable, args=args, childFDs=childFDs,
                    env=environ)


    def _driveWorker(self, worker, result, testCases, cooperate):
        """
        Drive a L{LocalWorkerAMP} instance, iterating the tests and calling
        C{run} for every one of them.

        @param worker: The L{LocalWorkerAMP} to drive.

        @param result: The global L{DistReporter} instance.

        @param testCases: The global list of tests to iterate.

        @param cooperate: The cooperate function to use, to be customized in
            tests.
        @type cooperate: C{function}

        @return: A C{Deferred} firing when all the tests are finished.
        """

        def resultErrback(error, case):
            result.original.addFailure(case, error)
            return error

        def task(case):
            d = worker.run(case, result)
            d.addErrback(resultErrback, case)
            return d

        return cooperate(task(case) for case in testCases).whenDone()


    def run(self, suite, reactor=None, cooperate=cooperate,
            untilFailure=False):
        """
        Spawn local worker processes and load tests. After that, run them.

        @param suite: A tests suite to be run.

        @param reactor: The reactor to use, to be customized in tests.
        @type reactor: A provider of
            L{twisted.internet.interfaces.IReactorProcess}

        @param cooperate: The cooperate function to use, to be customized in
            tests.
        @type cooperate: C{function}

        @param untilFailure: If C{True}, continue to run the tests until they
            fail.
        @type untilFailure: C{bool}.

        @return: The test result.
        @rtype: L{DistReporter}
        """
        if reactor is None:
            from twisted.internet import reactor
        result = self._makeResult()
        count = suite.countTestCases()
        self._stream.write("Running %d tests.\n" % (count,))

        if not count:
            # Take a shortcut if there is no test
            suite.run(result.original)
            self.writeResults(result)
            return result

        testDir, testDirLock = _unusedTestDirectory(
            FilePath(self._workingDirectory))
        workerNumber = min(count, self._workerNumber)
        ampWorkers = [LocalWorkerAMP() for x in range(workerNumber)]
        workers = self.createLocalWorkers(ampWorkers, testDir.path)
        processEndDeferreds = [worker.endDeferred for worker in workers]
        self.launchWorkerProcesses(reactor.spawnProcess, workers,
                                   self._workerArguments)

        def runTests():
            testCases = iter(list(_iterateTests(suite)))

            workerDeferreds = []
            for worker in ampWorkers:
                workerDeferreds.append(
                    self._driveWorker(worker, result, testCases,
                                      cooperate=cooperate))
            return DeferredList(workerDeferreds, consumeErrors=True,
                                fireOnOneErrback=True)

        stopping = []

        def nextRun(ign):
            self.writeResults(result)
            if not untilFailure:
                return
            if not result.wasSuccessful():
                return
            d = runTests()
            return d.addCallback(nextRun)

        def stop(ign):
            testDirLock.unlock()
            if not stopping:
                stopping.append(None)
                reactor.stop()

        def beforeShutDown():
            if not stopping:
                stopping.append(None)
                d = DeferredList(processEndDeferreds, consumeErrors=True)
                return d.addCallback(continueShutdown)

        def continueShutdown(ign):
            self.writeResults(result)
            return ign

        d = runTests()
        d.addCallback(nextRun)
        d.addBoth(stop)

        reactor.addSystemEventTrigger('before', 'shutdown', beforeShutDown)
        reactor.run()

        return result


    def runUntilFailure(self, suite):
        """
        Run the tests with local worker processes until they fail.

        @param suite: A tests suite to be run.
        """
        return self.run(suite, untilFailure=True)