text:
import cProfile
import io
import pstats

import matplotlib.pyplot as plt
import matplotlib
import numpy as np
import torch
from shapely.geometry import Point
from shapely.geometry.polygon import Polygon

import lettuce as lt
from lettuce import Lattice, D2Q9, Obstacle2D
from lettuce import UnitConversion, BounceBackBoundary, EquilibriumBoundaryPU

class Obstacles:
    def __init__(self, lattice, x, y, r, m, l):
        self.flow = Obstacle2D(
            resolution_x=x,
            resolution_y=y,
            reynolds_number=r,
            mach_number=m,
            lattice=lattice,
            char_length_lu=l
        )
        x, y = self.flow.grid
        self.x = self.flow.units.convert_length_to_lu(x)
        self.y = self.flow.units.convert_length_to_lu(y)
        self.conditions = []
        self.area = 0

    def setSquareObstacle(self, xpos, ypos, length):
        # Square:
        square = np.zeros(self.x.shape, dtype=bool)
        N = self.x.shape[0]
        M = self.x.shape[1]
        for i in range(N):
            for j in range(M):
                if xpos <= i <= xpos + length and ypos <= j <= ypos + length:
                    square[i, j] = True
                else:
                    square[i, j] = False
        condition = square
        self.area += length ** 2
        self.conditions.append(condition)

    def setCylinderObstacle(self, xpos, ypos, r):
        # Cylinder:
        condition = np.sqrt((self.x - xpos) ** 2 + (self.y - ypos) ** 2) <= r
        self.area += np.pi * (r ** 2)
        self.conditions.append(condition)

    def setRectObstacle(self, xpos, ypos, xlength, ylength):
        # Rectangle:
        rect = np.zeros(self.x.shape, dtype=bool)
        N = self.x.shape[0]
        M = self.x.shape[1]
        for i in range(N):
            for j in range(M):
                if xpos <= i <= xpos + xlength and ypos <= j <= ypos + ylength:
                    rect[i, j] = True
                else:
                    rect[i, j] = False
        condition = rect
        self.area += xlength * ylength
        self.conditions.append(condition)

    def setPolygonObstacle(self, polygon):
        poly = np.zeros(self.x.shape, dtype=bool)
        N = self.x.shape[0]
        M = self.x.shape[1]
        for i in range(N):
            for j in range(M):
                point = Point(i, j)
                poly[i, j] = point.within(polygon)
        condition = poly
        self.area += polygon.area
        self.conditions.append(condition)

    def setObstacle(self):
        for c in self.conditions:
            self.flow.mask[np.where(c)] = 1

    def getFlow(self):
        return self.flow

class Model:
    def __init__(self, d, l):
        self.lattice = None
        self.flow = None
        self.collision = None
        self.streaming = None
        self.simulation = None
        self.device = torch.device(d)
        self.stencil = l
        self.dtype = torch.float32
        self.obstacle = None

    def initialization(self, data):
        self.setupLattice()
        self.obstacle = Obstacles(self.lattice, data["X"], data["Y"], 100, 0.1, 10)
        # setup obstacle
        for c in data["Cylinder"]:
            self.obstacle.setCylinderObstacle(c[0], c[1], c[2])
        for c in data["Square"]:
            self.obstacle.setSquareObstacle(c[0], c[1], c[2])
        for c in data["Rectangle"]:
            self.obstacle.setRectObstacle(c[0], c[1], c[2], c[3])
        for c in data["Polygon"]:
            self.obstacle.setPolygonObstacle(Polygon(c))
        self.obstacle.setObstacle()
        print(self.obstacle.area)
        flow = self.obstacle.flow
        self.setupFlow(flow)
        self.setupCollision("BGKCollision")
        self.setupStreaming()
        self.setupSimulation()

    def setupLattice(self):
        self.lattice = lt.Lattice(self.stencil, device=self.device, dtype=self.dtype)

    def setupCollision(self, type):
        if type == "BGKCollision":
            self.collision = lt.BGKCollision(self.lattice, tau=self.flow.units.relaxation_parameter_lu)
        if type == "RegularizedCollision":
            self.collision = lt.RegularizedCollision(self.lattice, tau=self.flow.units.relaxation_parameter_lu)
        if type == "KBCCollision2D":
            self.collision = lt.KBCCollision2D(self.lattice, tau=self.flow.units.relaxation_parameter_lu)

    def setupStreaming(self):
        self.streaming = lt.StandardStreaming(self.lattice)

    def setupSimulation(self):
        self.simulation = lt.Simulation(flow=self.flow, lattice=self.lattice, collision=self.collision,
                                        streaming=self.streaming)

    def setupFlow(self, flow):
        self.flow = flow

    def simulationOutput(self):
        Energy = lt.IncompressibleKineticEnergy(self.lattice, self.flow)
        reporter = lt.ObservableReporter(Energy, interval=1000, out=None)
        self.simulation.reporters.append(reporter)
        self.simulation.reporters.append(
            lt.VTKReporter(self.lattice, self.flow, interval=100, filename_base="./output"))
        self.simulation.initialize_f_neq()
        mlups = self.simulation.step(num_steps=10000)
        print("Performance in MLUPS:", mlups)

    def processEnergy(self):
        energy = np.array(self.simulation.reporters[0].out)

        # Kinetic Energy
        plt.rcParams.update({'font.size': 16})
        plt.rc('font', family='serif')
        plt.rc('axes', axisbelow=True)
        plt.grid(linestyle="--")
        plt.plot(energy[:, 1], energy[:, 2])
        plt.title('Kinetic energy')
        plt.xlabel('Time', labelpad=12)
        plt.ylabel('Energy in physical units', labelpad=12)
        fig = matplotlib.pyplot.gcf()
        fig.set_size_inches(12, 8)
        fig.savefig('Kinetic Energy.pdf')
        # plt.show()

        # Plot
        plt.rcParams.update({'font.size': 16})
        plt.rc('font', family='serif')
        plt.rc('axes', axisbelow=True)
        u = self.flow.units.convert_velocity_to_pu(self.lattice.u(self.simulation.f)).numpy()
        u_norm = np.linalg.norm(u, axis=0)
        plt.imshow(u_norm)
        fig = matplotlib.pyplot.gcf()
        fig.set_size_inches(12, 8)
        fig.savefig('Flow.pdf')
        # plt.show()

def execute(data):
    pr = cProfile.Profile()
    pr.enable()
    # cuda
    # s = Model(torch.device("cuda:0"), lt.D2Q9)
    # cpu
    s = Model(torch.device("cpu"), lt.D2Q9)
    s.initialization(data)
    # s.setupLattice()
    # o = Obstacles(s.lattice, 501, 301, 100, 0.1, 10)
    # o.setCylinderObstacle(150, 150, 20)
    # o.setCylinderObstacle(300, 150, 20)
    # o.setObstacle()
    # flow = o.getFlow()
    #
    # s.setupFlow(flow)
    # s.setupCollision("BGKCollision")
    # s.setupStreaming()
    # s.setupSimulation()
    #
    s.simulationOutput()
    s.processEnergy()

    pr.disable()
    s = io.StringIO()

    ps = pstats.Stats(pr, stream=s).sort_stats('tottime')
    ps.print_stats()
    print(s.getvalue())

    with open('profile.txt', 'w+') as f:
        f.write(s.getvalue())


text:
"""
Cristiano G Castro

Functions for plotting a pmf
"""

import numpy
from Pmf import Hist
import matplotlib.pyplot as plt

def plot_histogram(hist, xlabel=None, ylabel=None):
    """
    Plots a histogram

    @type hist: Hist

    Args:
        hist: the histogram

    """
    (x,w) = hist.Render();

    plt.hist(x, bins = max(x), weights=w)

    if not xlabel is None: plt.xlabel(xlabel)
    if not ylabel is None: plt.ylabel(ylabel)
    plt.show()

ext: py

reason: import numpy


text:
import io
import json
import os
import librosa
import numpy as np
import tempfile
from pathlib import Path

try:
    import importlib.resources as pkg_resources
except ImportError:
    # Try backported to PY<37 `importlib_resources`.
    import importlib_resources as pkg_resources

import cv2

import ibm_boto3
from ibm_botocore.client import Config

from choirless_smiler.smiler import Smiler, load_landmarks

LANDMARKS_URL = "http://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2"
CACHE_DIR = "~/.smiler"

def main(args):

    cos = createCOSClient(args)

    if not cos:
        raise ValueError(f"could not create COS instance")

    bucket = args.get('preview_bucket')

    notification = args.get('notification', {})
    key = args.get('key', notification.get('object_name', ''))
    if key.endswith(".jpg"):
        return {"success": "true"}

    # Create a temp dir for our files to use
    with tempfile.TemporaryDirectory() as tmpdir:

        # download file to temp dir
        file_path = Path(tmpdir, key)
        new_path = file_path.with_name(f'{file_path.stem}.jpg')

        cos.download_file(bucket, key, str(file_path))

        with pkg_resources.path('choirless_smiler', 'smile_detector.tflite') as model_path:

            print("Loading landmarks")
            landmarks_path = load_landmarks(LANDMARKS_URL, CACHE_DIR)
            print("Landmarks loaded")

            smiler = Smiler(landmarks_path, model_path)

            print("Calculating threshold")
            fg = smiler.frame_generator(str(file_path))
            threshold = smiler.calc_threshold(fg, 0.95)
            print("Threshold calculated", threshold)

            def callback(frame, smile_score):
                cv2.imwrite(str(new_path), frame)
                cos.upload_file(str(new_path), bucket, str(new_path.name))
                args["snapshot_key"] = str(new_path.name)
                print("New best frame, score:", smile_score)

            print("Finding smiliest frame")
            fg = smiler.frame_generator(str(file_path))
            ffg = smiler.filter_frames(fg, threshold)
            smile_score, image = smiler.find_smiliest_frame(ffg, callback=callback)
            print("Smilest frame, score:", smile_score)

        return args

def createCOSClient(args):
    """
    Create a ibm_boto3.client using the connectivity information
    contained in args.

    :param args: action parameters
    :type args: dict
    :return: An ibm_boto3.client
    :rtype: ibm_boto3.client
    """

    # if a Cloud Object Storage endpoint parameter was specified
    # make sure the URL contains the https:// scheme or the COS
    # client cannot connect
    if args.get('endpoint') and not args['endpoint'].startswith('https://'):
        args['endpoint'] = 'https://{}'.format(args['endpoint'])

    # set the Cloud Object Storage endpoint
    endpoint = args.get('endpoint',
                        'https://s3.us.cloud-object-storage.appdomain.cloud')

    # extract Cloud Object Storage service credentials
    cos_creds = args.get('__bx_creds', {}).get('cloud-object-storage', {})

    # set Cloud Object Storage API key
    api_key_id = \
        args.get('apikey',
                 args.get('apiKeyId',
                          cos_creds.get('apikey',
                                        os.environ
                                        .get('__OW_IAM_NAMESPACE_API_KEY')
                                        or '')))

    if not api_key_id:
        # fatal error; it appears that no Cloud Object Storage instance
        # was bound to the action's package
        return None

    # set Cloud Object Storage instance id
    svc_instance_id = args.get('resource_instance_id',
                               args.get('serviceInstanceId',
                                        cos_creds.get('resource_instance_id',
                                                      '')))
    if not svc_instance_id:
        # fatal error; it appears that no Cloud Object Storage instance
        # was bound to the action's package
        return None

    ibm_auth_endpoint = args.get('ibmAuthEndpoint',
                                 'https://iam.cloud.ibm.com/identity/token')

    # Create a Cloud Object Storage client using the provided
    # connectivity information
    cos = ibm_boto3.client('s3',
                           ibm_api_key_id=api_key_id,
                           ibm_service_instance_id=svc_instance_id,
                           ibm_auth_endpoint=ibm_auth_endpoint,
                           config=Config(signature_version='oauth'),
                           endpoint_url=endpoint)

    # Return Cloud Object Storage client
    return cos

max_stars_repo_name: Choirless/renderer

max_issues_repo_name: krook/renderer

max_forks_repo_name: krook/renderer


text:
# -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2010-2018 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
"""\
The Starmap API
====================================

There are several good libraries to manage parallel programming, both
in the standard library and in third party packages. Since we are not
interested in reinventing the wheel, OpenQuake does not offer any new
parallel library; however, it does offer some glue code so that you
can use your library of choice. Currently threading, multiprocessing,
zmq and celery are supported. Moreover,
:mod:`openquake.baselib.parallel` offers some additional facilities
that make it easier to parallelize scientific computations,
i.e. embarrassing parallel problems.

Typically one wants to apply a callable to a list of arguments in
parallel rather then sequentially, and then combine together the
results. This is known as a `MapReduce` problem. As a simple example,
we will consider the problem of counting the letters in a text. Here is
how you can solve the problem sequentially:

>>> from itertools import starmap  # map a function with multiple arguments
>>> from functools import reduce  # reduce an iterable with a binary operator
>>> from operator import add  # addition function
>>> from collections import Counter  # callable doing the counting

>>> arglist = [('hello',), ('world',)]  # list of arguments
>>> results = starmap(Counter, arglist)  # iterator over the results
>>> res = reduce(add, results, Counter())  # aggregated counts

>>> sorted(res.items())  # counts per letter
[('d', 1), ('e', 1), ('h', 1), ('l', 3), ('o', 2), ('r', 1), ('w', 1)]

Here is how you can solve the problem in parallel by using
:class:`openquake.baselib.parallel.Starmap`:

>>> res2 = Starmap(Counter, arglist).reduce()
>>> assert res2 == res  # the same as before

As you see there are some notational advantages with respect to use
`itertools.starmap`. First of all, `Starmap` has a `reduce` method, so
there is no need to import `functools.reduce`; secondly, the `reduce`
method has sensible defaults:

1. the default aggregation function is `add`, so there is no need to specify it
2. the default accumulator is an empty accumulation dictionary (see
   :class:`openquake.baselib.AccumDict`) working as a `Counter`, so there
   is no need to specify it.

You can of course override the defaults, so if you really want to
return a `Counter` you can do

>>> res3 = Starmap(Counter, arglist).reduce(acc=Counter())

In the engine we use nearly always callables that return dictionaries
and we aggregate nearly always with the addition operator, so such
defaults are very convenient. You are encouraged to do the same, since we
found that approach to be very flexible. Typically in a scientific
application you will return a dictionary of numpy arrays.

The parallelization algorithm used by `Starmap` will depend on the
environment variable `OQ_DISTRIBUTE`. Here are the possibilities
available at the moment:

`OQ_DISTRIBUTE` not set or set to "processpool":
  use multiprocessing
`OQ_DISTRIBUTE` set to "no":
  disable the parallelization, useful for debugging
`OQ_DISTRIBUTE` set to "celery":
   use celery, useful if you have multiple machines in a cluster
`OQ_DISTRIBUTE` set tp "zmq"
   use the zmq concurrency mechanism (experimental)

There is also an `OQ_DISTRIBUTE` = "threadpool"; however the
performance of using threads instead of processes is normally bad for the
kind of applications we are interested in (CPU-dominated, which large
tasks such that the time to spawn a new process is negligible with
respect to the time to perform the task), so it is not recommended.

If you are using a pool, is always a good idea to cleanup resources at the end
with

>>> Starmap.shutdown()

`Starmap.shutdown` is always defined. It does nothing if there is
no pool, but it is still better to call it: in the future, you may change
idea and use another parallelization strategy requiring cleanup. In this
way your code is future-proof.

The Starmap.apply API
====================================

The `Starmap` class has a very convenient classmethod `Starmap.apply`
which is used in several places in the engine. `Starmap.apply` is useful
when you have a sequence of objects that you want to split in homogenous chunks
and then apply a callable to each chunk (in parallel). For instance, in the
letter counting example discussed before, `Starmap.apply` could
be used as follows:

>>> text = 'helloworld'  # sequence of characters
>>> res3 = Starmap.apply(Counter, (text,)).reduce()
>>> assert res3 == res

The API of `Starmap.apply` is designed to extend the one of `apply`,
a builtin of Python 2; the second argument is the tuple of arguments
passed to the first argument. The difference with `apply` is that
`Starmap.apply` returns a :class:`Starmap` object so that nothing is
actually done until you iterate on it (`reduce` is doing that).

How many chunks will be produced? That depends on the parameter
`concurrent_tasks`; it it is not passed, it has a default of 5 times
the number of cores in your machine - as returned by `os.cpu_count()` -
and `Starmap.apply` will try to produce a number of chunks close to
that number. The nice thing is that it is also possible to pass a
`weight` function. Suppose for instance that instead of a list of
letters you have a list of seismic sources: some sources requires a
long computation time (such as `ComplexFaultSources`), some requires a
short computation time (such as `PointSources`). By giving an heuristic
weight to the different sources it is possible to produce chunks with
nearly homogeneous weight; in particular `PointSource` tasks will
contain a lot more sources than tasks with `ComplexFaultSources`.

It is *essential* in large computations to have a homogeneous task
distribution, otherwise you will end up having a big task dominating
the computation time (i.e. you may have 1000 cores of which 999 are free,
having finished all the short tasks, but you have to wait for days for
the single core processing the slow task). The OpenQuake engine does
a great deal of work trying to split slow sources in more manageable
fast sources.
"""
import os
import sys
import mock
import time
import socket
import signal
import pickle
import inspect
import logging
import operator
import functools
import itertools
import traceback
import multiprocessing.dummy
import numpy
try:
    from setproctitle import setproctitle
except ImportError:
    def setproctitle(title):
        "Do nothing"

from openquake.baselib import hdf5, config
from openquake.baselib.zeromq import zmq, Socket
from openquake.baselib.performance import Monitor, virtual_memory
from openquake.baselib.general import (
    split_in_blocks, block_splitter, AccumDict, humansize)

cpu_count = multiprocessing.cpu_count()
OQ_DISTRIBUTE = os.environ.get('OQ_DISTRIBUTE', 'processpool').lower()
if OQ_DISTRIBUTE == 'futures':  # legacy name
    print('Warning: OQ_DISTRIBUTE=futures is deprecated', file=sys.stderr)
    OQ_DISTRIBUTE = os.environ['OQ_DISTRIBUTE'] = 'processpool'
if OQ_DISTRIBUTE not in ('no', 'processpool', 'threadpool', 'celery', 'zmq'):
    raise ValueError('Invalid oq_distribute=%s' % OQ_DISTRIBUTE)

def oq_distribute(task=None):
    """
    :returns: the value of OQ_DISTRIBUTE or 'processpool'
    """
    dist = os.environ.get('OQ_DISTRIBUTE', 'processpool').lower()
    read_access = getattr(task, 'read_access', True)
    if dist.startswith('celery') and not read_access:
        raise ValueError('You must configure the shared_dir in openquake.cfg '
                         'in order to be able to run %s with celery' %
                         task.__name__)
    return dist

def check_mem_usage(monitor=Monitor(),
                    soft_percent=None, hard_percent=None):
    """
    Display a warning if we are running out of memory

    :param int mem_percent: the memory limit as a percentage
    """
    soft_percent = soft_percent or config.memory.soft_mem_limit
    hard_percent = hard_percent or config.memory.hard_mem_limit
    used_mem_percent = virtual_memory().percent
    if used_mem_percent > hard_percent:
        raise MemoryError('Using more memory than allowed by configuration '
                          '(Used: %d%% / Allowed: %d%%)! Shutting down.' %
                          (used_mem_percent, hard_percent))
    elif used_mem_percent > soft_percent:
        hostname = socket.gethostname()
        logging.warn('Using over %d%% of the memory in %s!',
                     used_mem_percent, hostname)

class Pickled(object):
    """
    An utility to manually pickling/unpickling objects.
    The reason is that celery does not use the HIGHEST_PROTOCOL,
    so relying on celery is slower. Moreover Pickled instances
    have a nice string representation and length giving the size
    of the pickled bytestring.

    :param obj: the object to pickle
    """
    def __init__(self, obj):
        self.clsname = obj.__class__.__name__
        self.calc_id = str(getattr(obj, 'calc_id', ''))  # for monitors
        try:
            self.pik = pickle.dumps(obj, pickle.HIGHEST_PROTOCOL)
        except TypeError as exc:  # can't pickle, show the obj in the message
            raise TypeError('%s: %s' % (exc, obj))

    def __repr__(self):
        """String representation of the pickled object"""
        return '<Pickled %s %s %s>' % (
            self.clsname, self.calc_id, humansize(len(self)))

    def __len__(self):
        """Length of the pickled bytestring"""
        return len(self.pik)

    def unpickle(self):
        """Unpickle the underlying object"""
        return pickle.loads(self.pik)

def get_pickled_sizes(obj):
    """
    Return the pickled sizes of an object and its direct attributes,
    ordered by decreasing size. Here is an example:

    >> total_size, partial_sizes = get_pickled_sizes(Monitor(''))
    >> total_size
    345
    >> partial_sizes
    [('_procs', 214), ('exc', 4), ('mem', 4), ('start_time', 4),
    ('_start_time', 4), ('duration', 4)]

    Notice that the sizes depend on the operating system and the machine.
    """
    sizes = []
    attrs = getattr(obj, '__dict__',  {})
    for name, value in attrs.items():
        sizes.append((name, len(Pickled(value))))
    return len(Pickled(obj)), sorted(
        sizes, key=lambda pair: pair[1], reverse=True)

def pickle_sequence(objects):
    """
    Convert an iterable of objects into a list of pickled objects.
    If the iterable contains copies, the pickling will be done only once.
    If the iterable contains objects already pickled, they will not be
    pickled again.

    :param objects: a sequence of objects to pickle
    """
    cache = {}
    out = []
    for obj in objects:
        obj_id = id(obj)
        if obj_id not in cache:
            if isinstance(obj, Pickled):  # already pickled
                cache[obj_id] = obj
            else:  # pickle the object
                cache[obj_id] = Pickled(obj)
        out.append(cache[obj_id])
    return out

class Result(object):
    """
    :param val: value to return or exception instance
    :param mon: Monitor instance
    :param tb_str: traceback string (empty if there was no exception)
    """
    def __init__(self, val, mon, tb_str=''):
        self.pik = Pickled(val)
        self.mon = mon
        self.tb_str = tb_str

    def get(self):
        """
        Returns the underlying value or raise the underlying exception
        """
        with self.mon('unpickling %s' % self.mon.operation):
            val = self.pik.unpickle()
        if self.tb_str:
            etype = val.__class__
            msg = '\n%s%s: %s' % (self.tb_str, etype.__name__, val)
            if issubclass(etype, KeyError):
                raise RuntimeError(msg)  # nicer message
            else:
                raise etype(msg)
        return val

def safely_call(func, args):
    """
    Call the given function with the given arguments safely, i.e.
    by trapping the exceptions. Return a pair (result, exc_type)
    where exc_type is None if no exceptions occur, otherwise it
    is the exception class and the result is a string containing
    error message and traceback.

    :param func: the function to call
    :param args: the arguments
    """
    with Monitor('total ' + func.__name__, measuremem=True) as child:
        if args and hasattr(args[0], 'unpickle'):
            # args is a list of Pickled objects
            args = [a.unpickle() for a in args]
        if args and isinstance(args[-1], Monitor):
            mon = args[-1]
            mon.operation = func.__name__
            mon.children.append(child)  # child is a child of mon
            child.hdf5path = mon.hdf5path
        else:
            mon = child
        try:
            res = Result(func(*args), mon)
        except:
            _etype, exc, tb = sys.exc_info()
            res = Result(exc, mon, ''.join(traceback.format_tb(tb)))
    # FIXME: check_mem_usage is disabled here because it's causing
    # dead locks in threads when log messages are raised.
    # Check is done anyway in other parts of the code
    # further investigation is needed
    # check_mem_usage(mon)  # check if too much memory is used
    backurl = getattr(mon, 'backurl', None)
    zsocket = (Socket(backurl, zmq.PUSH, 'connect') if backurl
               else mock.MagicMock())  # do nothing
    with zsocket:
        zsocket.send(res)
    return zsocket.num_sent if backurl else res

if OQ_DISTRIBUTE.startswith('celery'):
    from celery.result import ResultSet
    from celery import Celery
    from celery.task import task
    app = Celery('openquake')
    app.config_from_object('openquake.engine.celeryconfig')
    safetask = task(safely_call, queue='celery')  # has to be global

class IterResult(object):
    """
    :param iresults:
        an iterator over Result objects
    :param taskname:
        the name of the task
    :param num_tasks:
        the total number of expected tasks
    :param progress:
        a logging function for the progress report
    :param sent:
        the number of bytes sent (0 if OQ_DISTRIBUTE=no)
    """
    def __init__(self, iresults, taskname, argnames, num_tasks, sent,
                 progress=logging.info):
        self.iresults = iresults
        self.name = taskname
        self.argnames = ' '.join(argnames)
        self.num_tasks = num_tasks
        self.sent = sent
        self.progress = progress
        self.received = []
        if self.num_tasks:
            self.log_percent = self._log_percent()
            next(self.log_percent)
        else:
            self.progress('No %s tasks were submitted', self.name)
        self.task_data_dt = numpy.dtype(
            [('taskno', numpy.uint32), ('weight', numpy.float32),
             ('duration', numpy.float32), ('received', numpy.int64)])
        self.progress('Sent %s of data in %s task(s)',
                      humansize(sent.sum()), num_tasks)

    def _log_percent(self):
        yield 0
        done = 1
        prev_percent = 0
        while done < self.num_tasks:
            percent = int(float(done) / self.num_tasks * 100)
            if percent > prev_percent:
                self.progress('%s %3d%%', self.name, percent)
                prev_percent = percent
            yield done
            done += 1
        self.progress('%s 100%%', self.name)
        yield done

    def __iter__(self):
        self.received = []
        if self.num_tasks == 0:
            return
        for result in self.iresults:
            check_mem_usage()  # log a warning if too much memory is used
            if isinstance(result, BaseException):
                # this happens with WorkerLostError with celery
                raise result
            elif isinstance(result, Result):
                val = result.get()
                self.received.append(len(result.pik))
            else:  # this should never happen
                raise ValueError(result)
            next(self.log_percent)
            if not self.name.startswith('_'):  # no info for private tasks
                self.save_task_info(result.mon)
            yield val

        if self.received:
            tot = sum(self.received)
            max_per_task = max(self.received)
            self.progress('Received %s of data, maximum per task %s',
                          humansize(tot), humansize(max_per_task))

    def save_task_info(self, mon):
        if mon.hdf5path:
            duration = mon.children[0].duration  # the task is the first child
            tup = (mon.task_no, mon.weight, duration, self.received[-1])
            data = numpy.array([tup], self.task_data_dt)
            hdf5.extend3(mon.hdf5path, 'task_info/' + self.name, data,
                         argnames=self.argnames, sent=self.sent)
        mon.flush()

    def reduce(self, agg=operator.add, acc=None):
        if acc is None:
            acc = AccumDict()
        for result in self:
            acc = agg(acc, result)
        return acc

    @classmethod
    def sum(cls, iresults):
        """
        Sum the data transfer information of a set of results
        """
        res = object.__new__(cls)
        res.received = []
        res.sent = 0
        for iresult in iresults:
            res.received.extend(iresult.received)
            res.sent += iresult.sent
            name = iresult.name.split('#', 1)[0]
            if hasattr(res, 'name'):
                assert res.name.split('#', 1)[0] == name, (res.name, name)
            else:
                res.name = iresult.name.split('#')[0]
        return res

def init_workers():
    """Waiting function, used to wake up the process pool"""
    setproctitle('oq-worker')
    # unregister raiseMasterKilled in oq-workers to avoid deadlock
    # since processes are terminated via pool.terminate()
    signal.signal(signal.SIGTERM, signal.SIG_DFL)
    # prctl is still useful (on Linux) to terminate all spawned processes
    # when master is killed via SIGKILL
    try:
        import prctl
    except ImportError:
        pass
    else:
        # if the parent dies, the children die
        prctl.set_pdeathsig(signal.SIGKILL)
    return os.getpid()

def _wakeup(sec, mon):
    """Waiting function, used to wake up the process pool"""
    time.sleep(sec)
    return os.getpid()

class Starmap(object):
    task_ids = []
    calc_id = None

    @classmethod
    def init(cls, poolsize=None, distribute=OQ_DISTRIBUTE):
        if distribute == 'processpool' and not hasattr(cls, 'pool'):
            cls.pool = multiprocessing.Pool(poolsize, init_workers)
            m = Monitor('wakeup')
            cls(_wakeup, [(.2, m) for _ in range(cls.pool._processes)])
        elif distribute == 'threadpool' and not hasattr(cls, 'pool'):
            cls.pool = multiprocessing.dummy.Pool(poolsize)

    @classmethod
    def shutdown(cls, poolsize=None):
        if hasattr(cls, 'pool'):
            cls.pool.close()
            cls.pool.terminate()
            cls.pool.join()
            delattr(cls, 'pool')

    @classmethod
    def apply(cls, task, args, concurrent_tasks=cpu_count * 3,
              maxweight=None, weight=lambda item: 1,
              key=lambda item: 'Unspecified', name=None, distribute=None):
        """
        Apply a task to a tuple of the form (sequence, \*other_args)
        by first splitting the sequence in chunks, according to the weight
        of the elements and possibly to a key (see :func:
        `openquake.baselib.general.split_in_blocks`).

        :param task: a task to run in parallel
        :param args: the arguments to be passed to the task function
        :param concurrent_tasks: hint about how many tasks to generate
        :param maxweight: if not None, used to split the tasks
        :param weight: function to extract the weight of an item in arg0
        :param key: function to extract the kind of an item in arg0
        :param name: name of the task to be used in the log
        :param distribute: if not given, inferred from OQ_DISTRIBUTE
        :returns: an :class:`IterResult` object
        """
        arg0 = args[0]  # this is assumed to be a sequence
        args = args[1:]
        if maxweight:
            chunks = block_splitter(arg0, maxweight, weight, key)
        else:
            chunks = split_in_blocks(arg0, concurrent_tasks or 1, weight, key)
        task_args = [(ch,) + args for ch in chunks]
        return cls(task, task_args, name, distribute).submit_all()

    def __init__(self, task_func, task_args, name=None, distribute=None):
        self.__class__.init(distribute=distribute or OQ_DISTRIBUTE)
        self.task_func = task_func
        self.name = name or task_func.__name__
        self.task_args = task_args
        if self.name.startswith('_'):  # secret task
            self.progress = lambda *args: None
        else:
            self.progress = logging.info
        self.distribute = distribute or oq_distribute(task_func)
        # a task can be a function, a class or an instance with a __call__
        if inspect.isfunction(task_func):
            self.argnames = inspect.getargspec(task_func).args
        elif inspect.isclass(task_func):
            self.argnames = inspect.getargspec(task_func.__init__).args[1:]
        else:  # instance with a __call__ method
            self.argnames = inspect.getargspec(task_func.__call__).args[1:]
        self.receiver = 'tcp://%s:%s' % (
            config.dbserver.host, config.zworkers.receiver_ports)
        self.sent = numpy.zeros(len(self.argnames))

    @property
    def num_tasks(self):
        """
        The number of tasks, if known, or the empty string otherwise.
        """
        try:
            return len(self.task_args)
        except TypeError:  # generators have no len
            return ''
        # NB: returning -1 breaks openquake.hazardlib.tests.calc.
        # hazard_curve_new_test.HazardCurvesTestCase02 :-(

    def _genargs(self, backurl=None, pickle=True):
        """
        Add .task_no and .weight to the monitor and yield back
        the arguments by pickling them.
        """
        for task_no, args in enumerate(self.task_args, 1):
            mon = args[-1]
            if isinstance(mon, Monitor):
                # add incremental task number and task weight
                mon.task_no = task_no
                mon.weight = getattr(args[0], 'weight', 1.)
                mon.backurl = backurl
                self.calc_id = getattr(mon, 'calc_id', None)
            if pickle:
                args = pickle_sequence(args)
                self.sent += numpy.array([len(p) for p in args])
            if task_no == 1:  # first time
                self.progress('Submitting %s "%s" tasks', self.num_tasks,
                              self.name)
            yield args

    def submit_all(self):
        """
        :returns: an IterResult object
        """
        if self.num_tasks == 1 or self.distribute == 'no':
            it = self._iter_sequential()
        elif self.distribute in ('processpool', 'threadpool'):
            it = self._iter_pool()
        elif self.distribute == 'celery':
            it = self._iter_celery()
        elif self.distribute == 'zmq':
            it = self._iter_zmq()
        num_tasks = next(it)
        return IterResult(it, self.name, self.argnames, num_tasks,
                          self.sent, self.progress)

    def reduce(self, agg=operator.add, acc=None):
        """
        Submit all tasks and reduce the results
        """
        return self.submit_all().reduce(agg, acc)

    def __iter__(self):
        return iter(self.submit_all())

    def _iter_sequential(self):
        self.progress('Executing "%s" in process', self.name)
        allargs = list(self._genargs(pickle=False))
        yield len(allargs)
        for args in allargs:
            yield safely_call(self.task_func, args)

    def _iter_pool(self):
        safefunc = functools.partial(safely_call, self.task_func)
        allargs = list(self._genargs())
        yield len(allargs)
        for res in self.pool.imap_unordered(safefunc, allargs):
            yield res

    def iter_native(self, results):
        for task_id, result_dict in ResultSet(results).iter_native():
            self.task_ids.remove(task_id)
            yield result_dict['result']

    def _iter_celery(self):
        with Socket(self.receiver, zmq.PULL, 'bind') as socket:
            logging.info('Using receiver %s', socket.backurl)
            results = []
            for piks in self._genargs(socket.backurl):
                res = safetask.delay(self.task_func, piks)
                # populating Starmap.task_ids, used in celery_cleanup
                self.task_ids.append(res.task_id)
                results.append(res)
            num_results = len(results)
            yield num_results
            it = self.iter_native(results)
            isocket = iter(socket)
            while num_results:
                res = next(isocket)
                if self.calc_id and self.calc_id != res.mon.calc_id:
                    logging.warn('Discarding a result from job %d, since this '
                                 'is job %d', res.mon.calc_id, self.calc_id)
                    continue
                err = next(it)
                if isinstance(err, Exception):  # TaskRevokedError
                    raise err
                num_results -= 1
                yield res

    def _iter_zmq(self):
        with Socket(self.receiver, zmq.PULL, 'bind') as socket:
            task_in_url = ('tcp://%(master_host)s:%(task_in_port)s' %
                           config.zworkers)
            with Socket(task_in_url, zmq.PUSH, 'connect') as sender:
                num_results = 0
                for args in self._genargs(socket.backurl):
                    sender.send((self.task_func, args))
                    num_results += 1
            yield num_results
            isocket = iter(socket)
            while num_results:
                res = next(isocket)
                if self.calc_id and self.calc_id != res.mon.calc_id:
                    logging.warn('Discarding a result from job %d, since this '
                                 'is job %d', res.mon.calc_id, self.calc_id)
                    continue
                num_results -= 1
                yield res

def sequential_apply(task, args, concurrent_tasks=cpu_count * 3,
                     weight=lambda item: 1, key=lambda item: 'Unspecified'):
    """
    Apply sequentially task to args by splitting args[0] in blocks
    """
    chunks = split_in_blocks(args[0], concurrent_tasks or 1, weight, key)
    task_args = [(ch,) + args[1:] for ch in chunks]
    return itertools.starmap(task, task_args)

max_stars_repo_name: gfzriesgos/shakyground-lfs

max_issues_repo_name: gfzriesgos/shakyground-lfs

max_forks_repo_name: gfzriesgos/shakyground-lfs


text:
# -*- coding: utf-8 -*-
import pandas as pd
from flask import Flask, jsonify, render_template
from yahoofinancials import YahooFinancials
import numpy as np
from datetime import date
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from sklearn import preprocessing

# identify how many days into the future do we want to predict
future = int(30)
# identify the stocks
tickers = ['CMCSA', 'V', 'LMT', 'NVDA', 'FB', 'NFLX', 'FOX', 'ORCL', 'TSLA', 'BA']
names_dict = {'CMCSA':'Comcast', 'V':'Visa', 'LMT':'Lockheed Martin', 'NVDA':'Nivdia', 'FB':'Facebook', 'NFLX':'Netflix', 'FOX':'21st Century Fox', 'ORCL':'Oracle', 'TSLA':'Tesla', 'BA':'Boeing'}
sent_dict = {'CMCSA':"Comcast Corporation was chosen because it is one of the largest entertainment companies in the world, and is also one of the largest providers in TV and is the largest ISP in the US. It’s also experienced some extreme volatility this current year, as it has hit both a record high price this year and is also at a lower price than where it started year to date.",
'V':'Visa is one of the largest multinational financial services in the world and is the second largest card payment organization in the world. It is also one of the 30 companies that the Dow Jones Industrial Average tracks, and has been consistently exceeding expectations in its reported quarterly earnings. Even though Visa currently is trending downward, it’s only trending negatively in the short-term, over the year it has grown tremendously and has only been trending upward since its inception. Lastly, because there is lower volatility in the closing prices, it may have been one of the reasons why our machine learning could score a relatively high accurate score.',
'LMT':"Lockheed Martin is a global security and aerospace company that employs approximately 100,000 people worldwide and is principally engaged in the research, design, development, manufacture, integration and sustainment of advanced technology systems, products and services. It was formed by the merger of Lockheed Corporation with Martin Marietta in March 1995. Lockheed Martin had $51 Billion in sales for 2017.",
'NVDA':'Nvidia is an American technology company based in Santa Clara, California.  It designs graphics processing units (GPUs) for the gaming and professional markets, as well as system on a chip units (SoCs) for the mobile computing and automotive market. Its primary GPU product line, labeled GeForce, is in direct competition with Advanced Micro Devices (AMD) Radeon products. Nvidia expanded its presence in the gaming industry with its handheld Shield Portable, Shield Tablet and Shield Android TV.',
'FB':'Facebook is a social media and networking website based in Menlo Park, California. Mark Zuckerberg and a group of Harvard students launched the platform in 2004, where it was limited to Harvard students. It eventually expanded its user base to other higher education schools around the Boston area and from there it was on its way to becoming the giant that it is today. Facebook applied for a $5 billion IPO in 2012 and opened with a sharing price of $38 per share. Today they are at the forefront of social media influence, acquiring Instagram in 2012 in addition to WhatsApp and Oculus VR.',
'NFLX':"Netflix is an American media services provider, headquartered in Los Gatos, California. Founded in 1997 by Reed Hastings and Marc Randolph in Scotts Valley, California, the company's primary business is its subscription-based streaming media service, which offers online streaming of a library of films and television programs including those produced in-house.",
'FOX':'Twenty-First Century Fox, Inc. is an American multinational mass media corporation that is based in Midtown Manhattan, New York City. It is one of the two companies formed from the 2013 spin-off of the publishing assets of News Corporation, as founded by Rupert Murdoch in 1979.',
'ORCL':'Oracle Corporation is an American multinational computer technology corporation headquartered in Redwood Shores, California.',
'TSLA': 'Tesla, Inc. is an American automotive and energy company based in Palo Alto, California. The company specializes in electric car manufacturing and, through its SolarCity subsidiary, in solar panel manufacturing. Tesla’s mission is to accelerate the world’s transition to sustainable energy',
'BA':'The Boeing Company is an American multinational corporation that designs, manufactures, and sells airplanes, rotorcraft, rockets, satellites, and missiles worldwide. The company also provides leasing and product support services.'
}
# identify the date interval
date1 = '2016-01-01'
date2 = str(date.today())

# adjclose is the same as close
# initialize empty list to append
ti = []
acc = []
pred = []
act = []
for ticker in tickers:
    dat = pd.DataFrame()
    yahoo_financials = YahooFinancials(ticker)
    result = yahoo_financials.get_historical_price_data(date1, date2, 'daily')
    df = pd.DataFrame(data=result[ticker]['prices'])
    df['ticker'] = ticker
    df.drop(columns=['close', 'date'], inplace=True)
    df.rename(columns={'formatted_date':'date'}, inplace=True)
    dat = pd.concat([dat, df], ignore_index=True)
    dat = dat[['ticker', 'date', 'open', 'adjclose', 'low', 'high', 'volume']]
    dat = dat[['adjclose']]
    # predicting the last n days
    act.append(dat['adjclose'].values[-future:])
    dat['prediction'] = dat[['adjclose']].shift(-future)

    # prepare X and y
    X = np.array(dat.drop(['prediction'], axis=1))
    X = preprocessing.scale(X)
    # set forecast to the last n rows
    X_forecast = X[-future:]
    # get rid of the last n nan rows
    X = X[:-future]
    # get y
    y = np.array(dat['prediction'])
    y = y[:-future]

    # train test split 80% train, 20% test
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state=42)

    # linear Regression
    clf = LinearRegression()
    clf.fit(X_train,y_train)
    # Testing
    accuracy = clf.score(X_test, y_test)
    # print(f"Model accuracy for {ticker}: {accuracy}")
    forecast = clf.predict(X_forecast)

    ti.append(ticker)
    acc.append(accuracy)
    pred.append(forecast)

df = pd.DataFrame(pred).T
df.columns = ti
dates = pd.date_range(end=pd.datetime.today(), periods=future).date.astype('str').tolist()
df['Date'] = dates

actual = pd.DataFrame(act).T
actual.columns = ti
dates = pd.date_range(end=pd.datetime.today(), periods=future).date.astype('str').tolist()
actual['Date'] = dates

acc_dic = {}
for i in range(10):
    acc_dic[tickers[i]] = acc[i]

print('data ready!')

# 0. Create app
app = Flask(__name__)

# 1. prediction
@app.route('/')
def Prediction():
    prediction_table = []
    for i in range(df.shape[0]):
        dic = {}
        for j in df.columns:
            dic[j] = df[j][i]
        prediction_table.append(dic)

    actual_table = []
    for i in range(actual.shape[0]):
        dic = {}
        for j in actual.columns:
            dic[j] = actual[j][i]
        actual_table.append(dic)
    return render_template("index.html", data = prediction_table, acc = acc_dic, act = actual_table, names = names_dict, sent = sent_dict)

# 2. Run App
if __name__ == '__main__':
    app.run(port = 5000, debug=True)


text:
# -*- coding: utf-8 -*-

"""
@author: krakowiakpawel9@gmail.com
@site: e-smartdata.org
"""

import pandas as pd
import numpy as np
import wget
import seaborn as sns
import matplotlib.pyplot as plt
sns.set()

# %%
"""
http://data.seattle.gov/
"""
url = ('https://data.seattle.gov/api/views/65db-xm6k/'
      'rows.csv?accessType=DOWNLOAD')

wget.download(url)

# %%

df = pd.read_csv('./bicycle.csv', index_col='Date', parse_dates=True)
df = df.sort_index()

# %%
df.columns = ['East', 'West']
df['Total'] = df.East + df.West

# %%
df.isnull().sum()
df[df['West'].isnull()]
df = df.dropna()

# %%
df.plot(alpha=0.6, color='red')

# %%
df.resample('W').sum().plot(style=[':', '--', '-'])
# %%
df.resample('D').sum().rolling(30).sum().plot(style=[':', '--', '-'])

# %%
hour_gr = df.groupby(df.index.time).mean()
hour_ticks = 4 * 60 * 60 * np.arange(6)
hour_gr.plot(style=[':', '--', '-'], xticks=hour_ticks)

# %%
df['weekend'] = np.where(df.index.weekday < 5, 'Weekday', 'Weekend')

by_weekend_time = df.groupby(['weekend', df.index.time]).mean()

# %%
fig, ax = plt.subplots(1, 2, figsize=(14, 5))
by_weekend_time.loc['Weekday'].plot(ax=ax[0], xticks=hour_ticks)
by_weekend_time.loc['Weekend'].plot(ax=ax[1], xticks=hour_ticks)


text:
import pytest
import numpy as np
from matplotlib import pyplot as plt

class Test_crf:
    @pytest.mark.mpl_image_compare
    def test_segmentationFromKeypoints(self):
        from supermariopy.crf import SegmentationFromKeypoints
        from supermariopy import imageutils
        from skimage import data

        n_keypoints = 10
        var = 0.05
        keypoints = np.stack(
            [np.linspace(-1, 1, n_keypoints), np.linspace(-1, 1, n_keypoints)], axis=1
        )

        img = data.astronaut()
        segmentation_algorithm = SegmentationFromKeypoints(var)
        labels = segmentation_algorithm(img, keypoints)
        labels_rgb = imageutils.make_colors(n_keypoints + 1)[labels]
        heatmaps = imageutils.keypoints_to_heatmaps(img.shape[:2], keypoints, var)
        heatmaps_rgb = imageutils.colorize_heatmaps(
            heatmaps[np.newaxis, ...], imageutils.make_colors(n_keypoints)
        )
        fig, axes = plt.subplots(1, 2, figsize=(8, 4))
        axes[0].imshow(labels_rgb)
        axes[0].set_axis_off()

        axes[1].imshow(np.squeeze(heatmaps_rgb))
        axes[1].set_axis_off()
        return fig

    def test_segmentationFromKeypoints_lowRangeError(self):
        from supermariopy.crf import SegmentationFromKeypoints
        from supermariopy import imageutils
        from skimage import data

        n_keypoints = 10
        var = 0.05
        keypoints = np.stack(
            [np.linspace(-1, 1, n_keypoints), np.linspace(-1, 1, n_keypoints)], axis=1
        )

        img = data.astronaut()
        img = imageutils.convert_range(img, [0, 255], [0, 1])
        segmentation_algorithm = SegmentationFromKeypoints(var)
        with pytest.warns(Warning):
            labels = segmentation_algorithm(img, keypoints)

max_stars_repo_name: tatevm/supermariopy

max_issues_repo_name: tatevm/supermariopy

max_forks_repo_name: tatevm/supermariopy


text:
'''
Copyright 2022 Airbus SAS

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
'''
mode: python; py-indent-offset: 4; tab-width: 8; coding: utf-8
'''
from copy import deepcopy
from multiprocessing import cpu_count

import pandas as pd
from numpy import array, ndarray, delete

from gemseo.algos.design_space import DesignSpace
from gemseo.core.scenario import Scenario
from gemseo.formulations.formulations_factory import MDOFormulationsFactory
from sos_trades_core.api import get_sos_logger
from sos_trades_core.execution_engine.data_manager import POSSIBLE_VALUES
from sos_trades_core.execution_engine.ns_manager import NS_SEP, NamespaceManager
from sos_trades_core.execution_engine.sos_discipline_builder import SoSDisciplineBuilder

class SoSScenario(SoSDisciplineBuilder, Scenario):
    '''
    Generic implementation of Scenario
    '''
    # Default values of algorithms

    # ontology information
    _ontology_data = {
        'label': 'sos_trades_core.execution_engine.sos_scenario',
        'type': 'Research',
        'source': 'SoSTrades Project',
        'validated': '',
        'validated_by': 'SoSTrades Project',
        'last_modification_date': '',
        'category': '',
        'definition': '',
        'icon': '',
        'version': '',
    }
    default_algo_options = {"ftol_rel": 3e-16,
                            "normalize_design_space": True,
                            "maxls": 100,
                            "maxcor": 50,
                            "pg_tol": 1.e-8,
                            "max_iter": 500,
                            "disp": 30}
    default_parallel_options = {'parallel': False,
                                'n_processes': cpu_count(),
                                'use_threading': False,
                                'wait_time_between_fork': 0}
    USER_GRAD = 'user'
    # Design space dataframe headers
    VARIABLES = "variable"
    VALUES = "value"
    UPPER_BOUND = "upper_bnd"
    LOWER_BOUND = "lower_bnd"
    TYPE = "type"
    ENABLE_VARIABLE_BOOL = "enable_variable"
    LIST_ACTIVATED_ELEM = "activated_elem"
    VARIABLE_TYPE = "variable_type"
    # To be defined in the heritage
    is_constraints = None
    INEQ_CONSTRAINTS = 'ineq_constraints'
    EQ_CONSTRAINTS = 'eq_constraints'
    # DESC_I/O
    PARALLEL_OPTIONS = 'parallel_options'
    # FD step
    FD_STEP = "fd_step"

    algo_dict = {}

#                                 'dataframe_descriptor': {VARIABLES: ('string', None, True),
#                                                          VALUES: ('array', None, True),
#                                                          LOWER_BOUND: ('array', None, True),
#                                                          UPPER_BOUND: ('array', None, True),
#                                                          ENABLE_VARIABLE_BOOL: ('bool', None, True),
#                                                          LIST_ACTIVATED_ELEM: ('list', None, True), },
#                                 'dataframe_edition_locked': False,
    DESC_IN = {'algo': {'type': 'string', 'structuring': True},
               'design_space': {'type': 'dataframe', 'structuring': True},
               'formulation': {'type': 'string', 'structuring': True},
               'maximize_objective': {'type': 'bool', 'structuring': True, 'default': False},
               'objective_name': {'type': 'string', 'structuring': True},
               'differentiation_method': {'type': 'string', 'default': Scenario.FINITE_DIFFERENCES,
                                          'possible_values': [USER_GRAD, Scenario.FINITE_DIFFERENCES,
                                                              Scenario.COMPLEX_STEP],
                                          'structuring': True},
               'fd_step': {'type': 'float', 'structuring': True, 'default': 1e-6},
               'algo_options': {'type': 'dict', 'dataframe_descriptor': {VARIABLES: ('string', None, False),
                                                                         VALUES: ('string', None, True)},
                                'dataframe_edition_locked': False,
                                'default': default_algo_options,
                                'structuring': True},
               PARALLEL_OPTIONS: {'type': 'dict',  # SoSDisciplineBuilder.OPTIONAL: True,
                                  'dataframe_descriptor': {VARIABLES: ('string', None, False),  # bool
                                                           VALUES: ('string', None, True)},
                                  #                                   'dataframe_descriptor': {'parallel': ('int', None, True), #bool
                                  #                                                          'n_processes': ('int', None, True),
                                  #                                                          'use_threading': ('int', None, True),#bool
                                  #                                                          'wait_time_between_fork': ('int', None, True)},
                                  'dataframe_edition_locked': False,
                                  'default': default_parallel_options,
                                  'structuring': True},
               'eval_mode': {'type': 'bool', 'default': False, POSSIBLE_VALUES: [True, False], 'structuring': True},
               'eval_jac': {'type': 'bool', 'default': False, POSSIBLE_VALUES: [True, False]},
               'execute_at_xopt': {'type': 'bool', 'default': True}}

    DESC_OUT = {'design_space_out': {'type': 'dataframe'}
                }

    def __init__(self, sos_name, ee, cls_builder):
        """
        Constructor
        """
        self.__factory = ee.factory
        self.cls_builder = cls_builder
        self.formulation = None
        self.maximize_objective = None
        self.opt_problem = None
        self._maturity = None

        self._reload(sos_name, ee)
        self.logger = get_sos_logger(f'{self.ee.logger.name}.SoSScenario')

        self.DESIGN_SPACE = 'design_space'
        self.FORMULATION = 'formulation'
        self.MAXIMIZE_OBJECTIVE = 'maximize_objective'
        self.OBJECTIVE_NAME = 'objective_name'
        self.FORMULATION_OPTIONS = 'formulation_options'

        #        self.SEARCH_PATHS = 'search_paths'
        self.SCENARIO_MANDATORY_FIELDS = [
            self.DESIGN_SPACE,
            self.FORMULATION,
            self.MAXIMIZE_OBJECTIVE,
            self.OBJECTIVE_NAME]
        #            self.SEARCH_PATHS]
        self.OPTIMAL_OBJNAME_SUFFIX = "opt"
        self.dict_desactivated_elem = {}
        self.activated_variables = []

    def _reload(self, sos_name, ee):
        """
        reload object
        """
        SoSDisciplineBuilder._reload(self, sos_name, ee)

    def build(self):
        """
        build of subdisciplines
        """
        # build and set sos_disciplines (if any)
        if len(self.cls_builder) != 0:
            old_current_discipline = self.ee.factory.current_discipline
            self.ee.factory.current_discipline = self
            # get the list of builders
            builder_list = self.cls_builder
            if not isinstance(self.cls_builder, list):
                builder_list = [self.cls_builder]
            # build the disciplines if not already built
            for builder in builder_list:
                disc = builder.build()
                if disc not in self.sos_disciplines:
                    self.ee.factory.add_discipline(disc)

                # append added disciplines to built_sos_disciplines for
                # disciplines cleaning
                if disc not in self.built_sos_disciplines:
                    self.built_sos_disciplines.append(disc)

            self.ee.factory.current_discipline = old_current_discipline

    def configure(self):
        """
        Configuration of SoSScenario, call to super Class and
        """
        self.configure_io()

        self.configure_execution()

        # Extract variables for eval analysis
        if self.sos_disciplines is not None and len(self.sos_disciplines) > 0:
            self.set_eval_possible_values()

        # update MDA flag to flush residuals between each mda run
        self._set_flush_submdas_to_true()

    def is_configured(self):
        """
        Return False if at least one sub discipline needs to be configured, True if not
        """
        return self.get_configure_status() and not self.check_structuring_variables_changes() and (
            self.get_disciplines_to_configure() == [])

    def setup_sos_disciplines(self):
        """
        Overload setup_sos_disciplines to create a dynamic desc_in
        """
        if self.ALGO_OPTIONS in self._data_in:
            algo_name = self.get_sosdisc_inputs(self.ALGO)
            algo_options = self.get_sosdisc_inputs(self.ALGO_OPTIONS)
            if algo_name is not None:
                default_dict = self.get_algo_options(algo_name)
                self._data_in[self.ALGO_OPTIONS][self.DEFAULT] = default_dict
                if algo_options is not None:
                    values_dict = deepcopy(default_dict)

                    for k in algo_options.keys():
                        if algo_options[k] != 'None' or not isinstance(algo_options[k], type(None)):
                            values_dict.update({k: algo_options[k]})

                    self._data_in[self.ALGO_OPTIONS][self.VALUE] = values_dict

        self.set_edition_inputs_if_eval_mode()

    def set_edition_inputs_if_eval_mode(self):
        '''
        if eval mode then algo and algo options will turn to not editable
        '''

        if 'eval_mode' in self._data_in:
            eval_mode = self.get_sosdisc_inputs('eval_mode')
            if eval_mode:
                self._data_in[self.ALGO][self.EDITABLE] = False
                self._data_in[self.ALGO_OPTIONS][self.EDITABLE] = False
                self._data_in[self.FORMULATION][self.EDITABLE] = False
                self._data_in[self.MAXIMIZE_OBJECTIVE][self.EDITABLE] = False
                self._data_in[self.PARALLEL_OPTIONS][self.EDITABLE] = False

                self._data_in[self.ALGO][self.OPTIONAL] = True
                self._data_in[self.ALGO_OPTIONS][self.OPTIONAL] = True
                self._data_in[self.FORMULATION][self.OPTIONAL] = True
                self._data_in[self.MAXIMIZE_OBJECTIVE][self.OPTIONAL] = True
                self._data_in[self.PARALLEL_OPTIONS][self.OPTIONAL] = True
            else:
                self._data_in['eval_jac'][self.VALUE] = False

    def pre_set_scenario(self):
        """
        prepare the set GEMS set_scenario method
        """
        design_space = None
        formulation = None
        obj_full_name = None
        maximize_objective = False

        dspace = self.get_sosdisc_inputs(self.DESIGN_SPACE)
        if dspace is not None:
            if any(type(design_variable).__name__ not in ['array', 'list'] for design_variable in
                   dspace['value'].tolist()):
                raise ValueError(
                    'A design variable must obligatory be an array')

            # build design space
            design_space = self.set_design_space()
            if design_space.variables_names:
                _, formulation, maximize_objective, obj_name = self.get_sosdisc_inputs(
                    self.SCENARIO_MANDATORY_FIELDS)

                # get full objective ids
                obj_name = self.get_sosdisc_inputs(self.OBJECTIVE_NAME)
                obj_full_name = self._update_names([obj_name])[0]

        return design_space, formulation, maximize_objective, obj_full_name

    def set_scenario(self):
        """
        set GEMS scenario, to be overloaded with each type of scenario (MDO, DOE, ...)
        """
        pass

    def set_design_space_for_complex_step(self):
        '''
        Set design space values to complex if the differentiation method is complex_step
        '''
        diff_method = self.get_sosdisc_inputs('differentiation_method')
        if diff_method == self.COMPLEX_STEP:
            dspace = deepcopy(self.opt_problem.design_space)
            curr_x = dspace._current_x
            for var in curr_x:
                curr_x[var] = curr_x[var].astype('complex128')
            self.opt_problem.design_space = dspace

    def update_default_coupling_inputs(self):
        '''
        Update default inputs of the couplings
        '''
        for disc in self.sos_disciplines:
            if disc.is_sos_coupling:
                self._set_default_inputs_from_dm(disc)

    def get_algo_options(self, algo_name):
        """
        Create default dict for algo options
        :param algo_name: the name of the algorithm
        :returns: dictionary with algo options default values
        """
        # TODO : add warning and log algo options

        default_dict = {}
        driver_lib = self._algo_factory.create(algo_name)
        driver_lib.init_options_grammar(algo_name)
        schema_dict = driver_lib.opt_grammar.schema.to_dict()
        properties = schema_dict.get(driver_lib.opt_grammar.PROPERTIES_FIELD)
        algo_options_keys = list(properties.keys())

        found_algo_names = [
            key for key in self.algo_dict.keys() if key in algo_name]
        if len(found_algo_names) == 1:
            key = found_algo_names[0]
            for algo_option in algo_options_keys:
                default_val = self.algo_dict[key].get(algo_option)
                if default_val is not None:
                    default_dict[algo_option] = default_val
        else:
            for algo_option in algo_options_keys:
                if algo_option in self.default_algo_options:
                    algo_default_val = self.default_algo_options.get(algo_option)
                    if algo_default_val is not None:
                        default_dict[algo_option] = algo_default_val

        return default_dict

    def run(self):
        '''
        Run method
        '''
        # TODO: to delete when MDA initialization is improved
        self.update_default_coupling_inputs()

        self.set_design_space_for_complex_step()

        eval_mode = self.get_sosdisc_inputs('eval_mode')
        if eval_mode:
            self.run_eval_mode()

        else:
            self.run_scenario()
        outputs = [discipline.get_output_data()
                   for discipline in self.sos_disciplines]
        for data in outputs:
            self.local_data.update(data)
        # store local data in datamanager
        self.update_dm_with_local_data()

    def run_scenario(self):
        '''
        Run the scenario and store last design_space
        '''
        pass

    def run_eval_mode(self):
        '''
        Run evaluate functions with the initial x
        '''
        eval_jac = self.get_sosdisc_inputs('eval_jac')
        design_space = self.get_sosdisc_inputs('design_space')

        self.opt_problem.evaluate_functions(
            eval_jac=eval_jac, normalize=False)
        # if eval mode design space was not modified
        self.store_sos_outputs_values(
            {'design_space_out': design_space}, update_dm=True)

    def _run_algorithm(self):
        """
        Runs the algo
        """
        pass

    def _set_flush_submdas_to_true(self):
        # update MDA flag to flush residuals between each mda run
        for disc in self.sos_disciplines:
            if disc.is_sos_coupling:
                if len(disc.sub_mda_list) > 0:
                    for sub_mda in disc.sub_mda_list:
                        sub_mda.reset_history_each_run = True

    def _set_default_inputs_from_dm(self, disc):
        """
        Based on dm values, default_inputs are set to mdachains,
        and default_inputs dtype is set to complex in case of complex_step gradient computation.
        """
        input_data = {}
        input_data_names = disc.get_input_data_names()
        for data_name in input_data_names:
            val = self.ee.dm.get_value(data_name)
            # for cases of early configure steps
            if val is not None:
                input_data[data_name] = val

        # convert sostrades types into numpy arrays
        # no need to update DM since call by SoSTrades
        input_data = disc._convert_new_type_into_array(var_dict=input_data)
        disc.mdo_chain.default_inputs.update(input_data)

        for disc in disc.sos_disciplines:
            if disc.is_sos_coupling:
                self._set_default_inputs_from_dm(disc)

    def configure_io(self):
        """
        Configure discipline  and all sub-disciplines
        """
        if self._data_in == {} or self.check_structuring_variables_changes():
            super().configure()

        disc_to_configure = self.get_disciplines_to_configure()

        if len(disc_to_configure) > 0:
            self.set_configure_status(False)
        else:
            self.set_children_cache_inputs()
            self.set_configure_status(True)

        for disc in disc_to_configure:
            disc.configure()

    def get_disciplines_to_configure(self):
        """
        Get sub disciplines list to configure
        """
        disc_to_configure = []
        for disc in self.sos_disciplines:
            if not disc.is_configured():
                disc_to_configure.append(disc)
        return disc_to_configure

    def configure_execution(self):
        """
        - configure GEMS grammar
        - set scenario
        """
        for disc in self.sos_disciplines:
            disc.update_gems_grammar_with_data_io()
        self.set_scenario()
        self.set_parallel_options()

    def _update_names(self, names):
        """
        if no dot in the name, it looks for the full name in the dm
        else we suppose that this is a full name that needs to be updated with current
        study name
        |!| it will NOT work for names with a dot in data_io...
        """
        local_names = []
        full_names = []
        for name in names:
            if NamespaceManager.NS_SEP not in name:
                local_names.append(name)
            else:
                full_names.append(name)
        return self.get_full_names(local_names) + \
            self._update_study_ns_in_varname(full_names)

    def set_diff_method(self):
        """
        Set differentiation method and send a WARNING
        if some linearization_mode are not coherent with diff_method
        """
        diff_method = self.get_sosdisc_inputs('differentiation_method')

        if diff_method in self.APPROX_MODES:
            for disc in self.sos_disciplines:
                if disc.linearization_mode != diff_method:
                    self.logger.warning(
                        f'The differentiation method "{diff_method}" will overload the linearization mode "{disc.linearization_mode}" ')

        fd_step = self.get_sosdisc_inputs(self.FD_STEP)
        Scenario.set_differentiation_method(
            self, diff_method, fd_step)

    def set_parallel_options(self):
        """
        sets parallel options for jacobian approximation
        """

        # update default options with user options
        user_options = self.get_sosdisc_inputs(self.PARALLEL_OPTIONS)
        if user_options is None:
            user_options = {}
        options = deepcopy(self.default_parallel_options)
        options.update(user_options)
        parallel = options.pop("parallel")
        # update problem options
        if self.formulation is not None:
            self.opt_problem = self.formulation.opt_problem
            self.opt_problem.parallel_differentiation = parallel
            self.opt_problem.parallel_differentiation_options = options

    def set_design_space(self):
        """
        reads design space (set_design_space)
        """

        dspace_df = self.get_sosdisc_inputs(self.DESIGN_SPACE)
        # update design space dv with full names
        dvs = list(dspace_df[self.VARIABLES])
        full_dvs = []
        dspace_dict_updated = {}

        for key in dvs:

            full_key_l = self.get_full_names([key])
            if len(full_key_l) > 0:
                full_key = full_key_l[0]
                full_dvs.append(full_key)
                # dspace_dict_updated[full_key] = dspace_df[key]
            else:
                self.logger.warning(f" missing design variable in dm : {key}")
        if len(full_dvs) == len(dvs):
            dspace_dict_updated = dspace_df.copy()
            dspace_dict_updated[self.VARIABLES] = full_dvs

            design_space = self.read_from_dataframe(dspace_dict_updated)

        else:

            design_space = DesignSpace()
        return design_space

    def read_from_dict(self, dp_dict):
        """Parses a dictionary to read the DesignSpace

        :param dp_dict : design space dictionary
        :returns:  the design space
        """
        design_space = DesignSpace()
        for key in dp_dict:
            print(key)
            if type(dp_dict[key]['value']) != list and type(dp_dict[key]['value']) != ndarray:
                name = key
                var_type = ['float']

                size = 1
                l_b = array([dp_dict[key]['lower_bnd']])
                u_b = array([dp_dict[key]['upper_bnd']])
                value = array([dp_dict[key]['value']])
            else:
                size = len(dp_dict[key]['value'])
                var_type = ['float'] * size

                name = key
                l_b = array(dp_dict[key]['lower_bnd'])
                u_b = array(dp_dict[key]['upper_bnd'])
                value = array(dp_dict[key]['value'])

            design_space.add_variable(name, size, var_type, l_b, u_b, value)
        return design_space

    def read_from_dataframe(self, df):
        """Parses a DataFrame to read the DesignSpace

        :param df : design space df
        :returns:  the design space
        """
        names = list(df[self.VARIABLES])
        values = list(df[self.VALUES])
        l_bounds = list(df[self.LOWER_BOUND])
        u_bounds = list(df[self.UPPER_BOUND])
        enabled_variable = list(df[self.ENABLE_VARIABLE_BOOL])
        list_activated_elem = list(df[self.LIST_ACTIVATED_ELEM])

        # looking for the optionnal variable type in the design space
        if self.VARIABLE_TYPE in df:
            var_types = df[self.VARIABLE_TYPE]
        else:
            # set to None for all variables if not exists
            var_types = [None] * len(names)

        design_space = DesignSpace()

        for dv, val, lb, ub, l_activated, enable_var, vtype in zip(names, values, l_bounds, u_bounds, list_activated_elem, enabled_variable, var_types):

            # check if variable is enabled to add it or not in the design var
            if enable_var:
                self.dict_desactivated_elem[dv] = {}

                if type(val) != list and type(val) != ndarray:
                    size = 1
                    var_type = ['float']
                    l_b = array([lb])
                    u_b = array([ub])
                    value = array([val])
                else:
                    # check if there is any False in l_activated
                    if not all(l_activated):
                        index_false = l_activated.index(False)
                        self.dict_desactivated_elem[dv] = {
                            'value': val[index_false], 'position': index_false}

                        val = delete(val, index_false)
                        lb = delete(lb, index_false)
                        ub = delete(ub, index_false)

                    size = len(val)
                    var_type = ['float'] * size
                    l_b = array(lb)
                    u_b = array(ub)
                    value = array(val)

                # 'automatic' var_type values are overwritten if filled by the user
                if vtype is not None:
                    var_type = vtype

                design_space.add_variable(
                    dv, size, var_type, l_b, u_b, value)
        return design_space

    def read_from_dataframe_new(self, df):
        """Parses a DataFrame to read the DesignSpace

        :param df : design space df
        :returns:  the design space
        """
        names = df[self.VARIABLES]
        values = df[self.VALUES]
        l_bounds = df[self.LOWER_BOUND]
        u_bounds = df[self.UPPER_BOUND]

        design_space = DesignSpace()
        for dv, val, lb, ub in zip(names, values, l_bounds, u_bounds):
            #             if [type(val), type(lb), type(ub)] == [str] * 3:
            #                 val = eval(val)
            #                 lb = eval(lb)
            #                 ub = eval(ub)
            name = dv
            if type(val) != list and type(val) != ndarray:
                size = 1
                var_type = ['float']
                l_b = array([lb])
                u_b = array([ub])
                value = array([val])
            else:
                size = len(val)
                var_type = ['float'] * size
                l_b = array(lb)
                u_b = array(ub)
                value = array(val)
            design_space.add_variable(name, size, var_type, l_b, u_b, value)
        return design_space

    def update_design_space_out(self):
        """
        Method to update design space with opt value
        """
        design_space = deepcopy(self.get_sosdisc_inputs(self.DESIGN_SPACE))
        l_variables = design_space[self.VARIABLES]
        for var in l_variables:
            full_name_var = self.get_full_names([var])[0]
            if full_name_var in self.activated_variables:
                value_x_opt = list(self.formulation.design_space._current_x.get(
                    full_name_var))
                if self.dict_desactivated_elem[full_name_var] != {}:
                    # insert a desactivated element
                    value_x_opt.insert(
                        self.dict_desactivated_elem[full_name_var]['position'],
                        self.dict_desactivated_elem[full_name_var]['value'])

                design_space.loc[design_space[self.VARIABLES] == var, self.VALUE] = pd.Series(
                    [value_x_opt] * len(design_space))

        self.store_sos_outputs_values(
            {'design_space_out': design_space})

    # GEMSEO overload
    def _update_input_grammar(self):
        self.update_gems_grammar_with_data_io()
        Scenario._update_input_grammar(self)

    def get_maturity(self):
        ref_dict_maturity = deepcopy(self.dict_maturity_ref)
        for discipline in self.sos_disciplines:
            disc_maturity = discipline.get_maturity()

            if isinstance(disc_maturity, dict):
                for m_k, m_v in ref_dict_maturity.items():
                    if m_v != disc_maturity[m_k]:
                        ref_dict_maturity[m_k] += disc_maturity[m_k]
            elif disc_maturity in ref_dict_maturity:
                ref_dict_maturity[disc_maturity] += 1

        self._maturity = ref_dict_maturity
        return self._maturity

    def set_eval_possible_values(self):

        analyzed_disc = self.sos_disciplines
        possible_out_values = self.fill_possible_values(
            analyzed_disc)  # possible_in_values

        possible_out_values = self.find_possible_values(
            analyzed_disc, possible_out_values)  # possible_in_values

        # Take only unique values in the list
        possible_out_values = list(set(possible_out_values))

        # Fill the possible_values of obj and constraints
        self.dm.set_data(f'{self.get_disc_full_name()}.{self.OBJECTIVE_NAME}',
                         self.POSSIBLE_VALUES, possible_out_values)

        if self.is_constraints:
            self.dm.set_data(f'{self.get_disc_full_name()}.{self.INEQ_CONSTRAINTS}',
                             self.POSSIBLE_VALUES, possible_out_values)
            self.dm.set_data(f'{self.get_disc_full_name()}.{self.EQ_CONSTRAINTS}',
                             self.POSSIBLE_VALUES, possible_out_values)
        # fill the possible values of algos
        self._init_algo_factory()
        avail_algos = self._algo_factory.algorithms
        self.dm.set_data(f'{self.get_disc_full_name()}.{self.ALGO}',
                         self.POSSIBLE_VALUES, avail_algos)
        # fill the possible values of formulations
        self._form_factory = MDOFormulationsFactory()
        avail_formulations = self._form_factory.formulations
        self.dm.set_data(f'{self.get_disc_full_name()}.{self.FORMULATION}',
                         self.POSSIBLE_VALUES, avail_formulations)
        # fill the possible values of maximize_objective
        self.dm.set_data(f'{self.get_disc_full_name()}.{self.MAXIMIZE_OBJECTIVE}',
                         self.POSSIBLE_VALUES, [False, True])

    # -- Set possible design variables and objevtives
    # adapted from soseval
    # TODO: find a better way to select constraints and objectives

    def find_possible_values(
            self, sos_disciplines, possible_out_values):  # possible_in_values
        """
            Run through all disciplines and sublevels
            to find possible values for eval_inputs and eval_outputs
        """
        if len(sos_disciplines) != 0:
            for disc in sos_disciplines:
                sub_out_values = self.fill_possible_values(
                    [disc])  # sub_in_values
                #                 possible_in_values.extend(sub_in_values)
                possible_out_values.extend(sub_out_values)
                self.find_possible_values(
                    disc.sos_disciplines, possible_out_values)  # possible_in_values

        return possible_out_values  # possible_in_values

    def fill_possible_values(self, sos_disciplines):
        """
            Fill possible values lists for eval inputs and outputs
            an input variable must be a float coming from a data_in of a discipline in all the process
            and not a default variable
            an output variable must be any data from a data_out discipline
        """
        # poss_in_values = []
        poss_out_values = []
        for disc in sos_disciplines:
            #             for data_in_key in disc.get_input_data_names(): #disc._data_in.keys():
            #                 is_float = disc._data_in[data_in_key.split(NS_SEP)[-1]][self.TYPE] == 'float'
            #                 in_coupling_numerical = data_in_key in SoSCoupling.DEFAULT_NUMERICAL_PARAM
            #                 if not in_coupling_numerical: #is_float and
            #                     # Caution ! This won't work for variables with points in name
            #                     # as for ac_model
            #                     poss_in_values.append(data_in_key)
            for data_out_key in disc.get_output_data_names():  # disc._data_out.keys():
                # Caution ! This won't work for variables with points in name
                # as for ac_model
                data_out_key = data_out_key.split(NS_SEP)[-1]
                poss_out_values.append(data_out_key)

        return poss_out_values  # poss_in_values

    def __str__(self):
        """
        Summarize results for display

        :returns: string summarizing results
        """
        msg = ""
        if hasattr(self, "disciplines"):
            msg = self.__class__.__name__ + ":\nDisciplines: "
            disc_names = [disc.name
                          for disc in self.disciplines]  # pylint: disable=E1101
            msg += " ".join(disc_names)
            msg += "\nMDOFormulation: "  # We keep MDO here has is done in gemseo
            msg += self.formulation.__class__.__name__
            msg += "\nAlgorithm: "
            msg += str(self.get_sosdisc_inputs(self.ALGO)) + "\n"

        return msg

    def get_full_names(self, names):
        '''
        get full names of ineq_names and obj_names
        '''
        full_names = []
        for i_name in names:
            full_id_l = self.dm.get_all_namespaces_from_var_name(i_name)
            if full_id_l != []:
                if len(full_id_l) > 1:
                    # full_id = full_id_l[0]
                    full_id = self.get_scenario_lagr(full_id_l)
                else:
                    full_id = full_id_l[0]
                full_names.append(full_id)

        return full_names

    def get_algo_options_dict(self):
        """
        Method to get the dictionnary of algo options from dataframe so that it is in GEMS format
        """
        algo_options_df = self.get_sosdisc_inputs('algo_options')
        return algo_options_df.to_dict('records')[0]

max_stars_repo_name: os-climate/sostrades-core

max_issues_repo_name: os-climate/sostrades-core

max_forks_repo_name: os-climate/sostrades-core

reason: from numpy


text:
import os
import numpy as np

step1 = 0
step2 = 1
step3 = 0

'''
1. This will create seperate UC and OC Consistent conenctomes that contains
for each link, a score that specifies in how many preproc pipelines the link occured.
'''
# Set the ABIDE version you want to work on
ABIDE = 1
if step1:
    print("========================Step 1=====================================")
    print("Step 1: This will create seperate UC and OC Consistent conenctomes that contains \
    for each link, a score that specifies in how many preproc pipelines the link occured.")
    cmd = "python3 find_consistent_connectome.py %s"%ABIDE
    print(cmd)
    os.system(cmd)
    print("====================================================================")

#  Output filenames
out_file_OC='ABIDE%s/combined_brain_OC_ABIDE%s.nii.gz'%(ABIDE, ABIDE)
out_file_UC='ABIDE%s/combined_brain_UC_ABIDE%s.nii.gz'%(ABIDE, ABIDE)

'''
2. Map the above created OC and OC files to full brain (BN + CBLM) atlas and create
a CSV file. Use Multiple UC and OC thresholds. (8,8), (10,10) ... (16,16) and
create multiple CSV files.
'''
if step2:
    print("========================Step 2======================================")
    print("2. Map the above created OC and OC files to full brain (BN + CBLM) atlas and create \
    a CSV file. Use Multiple UC and OC thresholds. (8,8), (10,10) ... (16,16) and \
    create multiple CSV files.")
    #  Change the path to link_mapping_to_atlas folder
    os.chdir('link_mapping_to_atlas')
    print('Changed the directory to : ',os.getcwd())
    UC_brain_path = '../' + out_file_UC
    OC_brain_path = '../' + out_file_OC
    out_file_prefix = '../ABIDE%s/map_links_to_atlas/ABIDE%s_links_atlas_map'%(ABIDE, ABIDE)
    cmd = "python3 map_links_to_atlas.py %s %s %s"%(UC_brain_path, OC_brain_path, out_file_prefix)
    os.system(cmd)
    print(cmd)
    os.chdir('..') # Change the directory back to original
    print("====================================================================")

'''
3. Manually create pivot tables for each of the files created above.
Then run the follwing script to find the links common with the links of review
and create a new csv.
'''
if step3:
    print("====================================================================")
    #  Change the path to link_mapping_to_atlas folder
    os.chdir('link_mapping_to_atlas')
    print('Changed the directory to : ',os.getcwd())
    #  Using the same as used in Step 2 TODO: Modify script to pass it as args in Step 2
    # thresh_UC  = np.arange(8,17,2)
    # thresh_OC  = np.arange(8,17,2)
    thresh_UC = [1]
    thresh_OC = [1]

    thresh_UC_OC_list = list(zip(thresh_UC, thresh_OC))

    in_file1 = '../csv_input/BN_regions_review_links.csv'
    for thresh_UC_OC in thresh_UC_OC_list:
        in_file2 ='../ABIDE%s/map_links_to_atlas_pivot_tables/ABIDE%s_links_atlas_map_UC%s_OC%s\ -\ Pivot\ Table\ 1.csv'%(ABIDE,ABIDE,thresh_UC_OC[0],thresh_UC_OC[1])
        out_file_path = '../ABIDE%s/ABIDE%s_review_common_combined/ABIDE%s_review_common_combined_UC%s_OC%s.csv'%(ABIDE,ABIDE,ABIDE,thresh_UC_OC[0],thresh_UC_OC[1])

        cmd = "python3 find_common_links.py %s %s %s"%(in_file1, in_file2, out_file_path)
        print(cmd)
        os.system(cmd)

    os.chdir('..') # Change the directory back to original
    print("====================================================================")

'''
4. For each of the above creatd common file, Calculate the precision and recall
and then calculate the F-score.

Precision:
---------
a = Count of all the ABIDE links that have atleast one end point in review BN regions
b = Count of all the ABIDE links that were consistent (Not just common) with the review BN regions

Note: A replicated ABIDE link is consistent by default if the review link is inconsistent
precision (p) = b/a

Recall:
--------

c = Count of All the consistent replicated BN region links = b
d = Count of total BN review links

Note: A replicated ABIDE link is consistent by default if the review link is inconsistent

recall (r) = c/d (or) b/d

F-Score
------

f = 1 / ( (1/p) + (1/r) )

'''

max_stars_repo_name: varun-invent/feature-extractor

max_issues_repo_name: varun-invent/feature-extractor

max_forks_repo_name: varun-invent/feature-extractor


text:
# -*- coding: utf-8 -*-

# Copyright (c) 2016-2022 by University of Kassel and Fraunhofer Institute for Energy Economics
# and Energy System Technology (IEE), Kassel. All rights reserved.

import numpy as np
import pytest

import pandapower as pp
import pandapower.networks as nw
import copy
from pandapower.auxiliary import _check_connectivity, _add_ppc_options
from pandapower.pd2ppc import _pd2ppc
from pandapower.test.consistency_checks import rundcpp_with_consistency_checks
from pandapower.test.loadflow.result_test_network_generator import result_test_network_generator_dcpp
from pandapower.test.toolbox import add_grid_connection, create_test_line, assert_net_equal

def test_rundcpp_init():
    net = pp.create_empty_network()
    b1, b2, l1 = add_grid_connection(net)
    b3 = pp.create_bus(net, vn_kv=0.4)
    tidx = pp.create_transformer(net, hv_bus=b2, lv_bus=b3, std_type="0.25 MVA 20/0.4 kV")
    pp.rundcpp(net)

def test_rundcpp_init_auxiliary_buses():
    net = pp.create_empty_network()
    b1, b2, l1 = add_grid_connection(net, vn_kv=110.)
    b3 = pp.create_bus(net, vn_kv=20.)
    b4 = pp.create_bus(net, vn_kv=10.)
    tidx = pp.create_transformer3w(net, b2, b3, b4, std_type='63/25/38 MVA 110/20/10 kV')
    pp.create_load(net, b3, p_mw=5)
    pp.create_load(net, b4, p_mw=5)
    pp.create_xward(net, b4, 1, 1, 1, 1, 0.1, 0.1, 1.0)
    net.trafo3w.shift_lv_degree.at[tidx] = 120
    net.trafo3w.shift_mv_degree.at[tidx] = 80
    pp.rundcpp(net)
    va = net.res_bus.va_degree.at[b2]
    pp.rundcpp(net)
    assert np.allclose(va - net.trafo3w.shift_mv_degree.at[tidx], net.res_bus.va_degree.at[b3],
                       atol=2)
    assert np.allclose(va - net.trafo3w.shift_lv_degree.at[tidx], net.res_bus.va_degree.at[b4],
                       atol=2)

# ToDo: Bugs because of float datatypes -> Check Travis on linux machines...
def test_result_iter():
    for net in result_test_network_generator_dcpp():
        try:
            rundcpp_with_consistency_checks(net)
        except (AssertionError):
            raise UserWarning("Consistency Error after adding %s" % net.last_added_case)
        except(pp.LoadflowNotConverged):
            raise UserWarning("Power flow did not converge after adding %s" % net.last_added_case)

def test_two_open_switches():
    net = pp.create_empty_network()
    b1, b2, l1 = add_grid_connection(net)
    b3 = pp.create_bus(net, vn_kv=20.)
    l2 = create_test_line(net, b2, b3)
    create_test_line(net, b3, b1)
    pp.create_switch(net, b2, l2, et="l", closed=False)
    pp.create_switch(net, b3, l2, et="l", closed=False)
    pp.rundcpp(net)
    assert np.isnan(net.res_line.i_ka.at[l2]) or net.res_line.i_ka.at[l2] == 0

def get_isolated(net):
    net._options = {}
    _add_ppc_options(net, calculate_voltage_angles=False,
                     trafo_model="t", check_connectivity=False,
                     mode="pf", r_switch=0.0, init="flat",
                     enforce_q_lims=False, recycle=None)
    ppc, ppci = _pd2ppc(net)
    return _check_connectivity(ppc)

def test_test_sn_mva():
    test_net_gen1 = result_test_network_generator_dcpp(sn_mva=1)
    test_net_gen2 = result_test_network_generator_dcpp(sn_mva=2)
    for net1, net2 in zip(test_net_gen1, test_net_gen2):
        pp.rundcpp(net1)
        pp.rundcpp(net2)
        try:
            assert_net_equal(net1, net2, exclude_elms=["sn_mva"])
        except:
            raise UserWarning("Result difference due to sn_mva after adding %s" % net1.last_added_case)

def test_single_bus_network():
    net = pp.create_empty_network()
    b = pp.create_bus(net, vn_kv=20.)
    pp.create_ext_grid(net, b)

    pp.runpp(net)
    assert net.converged

    pp.rundcpp(net)
    assert net.converged

def test_missing_gen():
    net = nw.case4gs()
    pp.rundcpp(net)
    res_gen = copy.deepcopy(net.res_gen.values)
    net.pop("res_gen")
    pp.rundcpp(net)
    assert np.allclose(net.res_gen.values, res_gen, equal_nan=True)

if __name__ == "__main__":
    pytest.main([__file__, "-xs"])


text:
import time
import numpy as np
from threading import Event
from utils import normalize_vector, get_brightness_char, clear_console
from status import Status

class Scene:
	def __init__(self, obj, light_source, camera, fps=5):
		# TODO add possiblity for multiple objects
		self.obj = obj
		self.light_source = light_source
		self.camera = camera
		if not (type(fps) == int and fps >= 1):
			raise Exception("FPS value must be an int greater or equal to 1, instead got '{}'".format(fps))
		self.fps = fps

	def __compute_illumination(self):
		self.brightnesses = [self.light_source.compute_brightness_on_point(self.camera.camera_point, point, normal) for point, normal in zip(self.obj.points, self.obj.normals)]

	def __compute_pixels(self):
		pixels = self.camera.plane.get_fresh_init_pixels()

		# only project the points on the camera plane that are visible to the camera, so we filter out the points with negative brightness
		points_with_positive_brightness = filter(lambda x: x[1] > 0, zip(self.obj.points, self.brightnesses))

		for point, brightness in points_with_positive_brightness:
			point_to_camera_vector = self.camera.camera_point - point
			distance_to_point = np.linalg.norm(point_to_camera_vector)

			intersection_on_plane = self.camera.plane.find_intersection(self.camera.camera_point, point_to_camera_vector)
			pixel = self.camera.plane.clip_point_to_pixel(intersection_on_plane)

			if pixel != None and distance_to_point < pixels[pixel][1]:
				pixels[pixel] = (intersection_on_plane, distance_to_point, brightness)

		return pixels

	def __print_pixels(self, pixels):
		clear_console()
		text_to_print = ""
		for i, (_, _, brightness) in pixels.items():
			if i % self.camera.plane.nb_pixel_width == 0 and i > 0:
				text_to_print += "\n"
			text_to_print += get_brightness_char(brightness)
		print(text_to_print)

	def __update_scene(self):
		self.__compute_illumination()
		pixels = self.__compute_pixels()
		self.__print_pixels(pixels)

	def modify_camera(self, camera):
		self.camera = camera

	def modify_light_source(self, light_source):
		self.light_source = light_source

	def start(self, static=False):
		if static:
			self.__update_scene()
		else:
			self.status = Status.RUN
			self.statusChangeEvent = Event()
			while True:
				if self.status == Status.STOP:
					break
				elif self.status == Status.PAUSE:
					self.__update_scene()
					# wait until self.status changes
					self.statusChangeEvent.wait()
				elif self.status == Status.RUN:
					before_comp_time = time.time()
					self.__update_scene()
					computation_time = time.time() - before_comp_time
					sleep_time = (1 / self.fps) - computation_time
				else:
					raise Exception("Unknown Status '{}'".format(self.status))

				time.sleep(max(sleep_time, 0))

	def move_camera(self, move):
		self.camera.move(move)

	def change_obj(self, new_obj):
		self.obj = new_obj

	def increment_light_source_intensity(self):
		self.light_source.increment_intensity()

	def decrement_light_source_intensity(self):
		self.light_source.decrement_intensity()

max_stars_repo_name: lucaalbinati/Torus

max_issues_repo_name: lucaalbinati/torus-engine

max_forks_repo_name: lucaalbinati/torus-engine