text:
#-*- coding: utf-8 -*-

""" EOSS catalog system
 functionality for the references endpoint
"""
import psycopg2

__author__ = "Thilo Wehrmann, Steffen Gebhardt"
__copyright__ = "Copyright 2016, EOSS GmbH"
__credits__ = ["Thilo Wehrmann", "Steffen Gebhardt"]
__license__ = "GPL"
__version__ = "1.0.0"
__maintainer__ = "Thilo Wehrmann"
__email__ = "twehrmann@eoss.cloud"
__status__ = "Production"

import ujson
import logging
import time

import falcon
import numpy
from fuzzywuzzy import fuzz
from shapely.geometry import Polygon

from client.services.static_maps import j2_env
from api_logging import logger
from .db_calls import Persistance
from .tools import make_GeoJson, get_base_url, can_zip_response, compress_body

THRES = 80
PRIORITY = {9: 0, 12: 1, 14: 2}  # TODO: Should be part of spatialreferencetype table , 13: 5

class Reference():
    """
    sample: http://localhost:8000/reference/9/73.json (e.g. France)
            http://localhost:8000/reference/9/73.geojson
            http://localhost:8000/reference/9/73.html
            http://localhost:8000/reference/9/all.html?bbox=6,45,12,55

            http://localhost:8000/reference/search/count?entity_name=puebla
    """

    # cors = cors

    def __init__(self, my_router):
        """
        Load all reference objects and reference types in memory for faster lookup

        :param my_router:
        """
        self.router = my_router
        self.logger = logging.getLogger('eoss.' + __name__)
        self.ref_objects = Persistance().get_all_references()
        self.ref_groups = dict()
        for obj in Persistance().get_all_reference_types():
            self.ref_groups[obj.id] = obj

    def on_get(self, req, resp, group_id, reference_id, format):
        """Handles GET requests

        """
        logger.info('[GET] /reference/%s/%s.%s' % (group_id, reference_id, format))

        start_time = time.time()
        results = dict()
        minx, miny, maxx, maxy = None, None, None, None

        if reference_id == 'all' and 'bbox' in req.params:
            minx, miny, maxx, maxy = [float(x) for x in req.params['bbox']]
            polygon = Polygon([(minx, miny), (minx, maxy), (maxx, maxy), (maxx, miny), (minx, miny)])

            ref_objects = Persistance().get_reference_by_groupid_polygon(group_id, polygon.wkt)
        elif reference_id != 'all':
            ref_objects = Persistance().get_reference_by_groupid_reference_id(group_id, reference_id)

        else:
            description = 'Please specify entity_name OR bbox with request, given %s:%s.' % (reference_id, req.params.get('bbox'))
            raise falcon.HTTPBadRequest('DateFormat', description,
                                        href='http://docs.example.com/auth')

        results['counter'] = ref_objects.count()

        geoms, attrs = list(), list()
        extents = list()
        for ref_objs in ref_objects.all():
            ref_obj, geojson, extent = ref_objs
            extents.append(ujson.loads(extent))
            geoms.append(ujson.loads(geojson))
            attrs.append({"ref_name": ref_obj.ref_name,
                          "reference_id": ref_obj.ref_id,
                          'group_id': self.ref_groups[ref_obj.referencetype_id].id,
                          'group_name': self.ref_groups[ref_obj.referencetype_id].name,
                          'group_description': self.ref_groups[ref_obj.referencetype_id].description
                          })

        results['geojson'] = make_GeoJson(geoms, attrs)
        # return plain geojson object

        results['processing_time'] = time.time() - start_time
        content_type = 'application/json'
        if format == 'json':
            del results['geojson']
            results['attributes']= attrs
        elif format == 'geojson':
            results = results['geojson']
        elif format == 'html':
            if minx != None and miny != None and maxx != None and maxy != None:
                global_extent = [[miny, minx], [maxy, maxx]]
            else:
                mins = list()
                maxs = list()
                for ext in extents:
                    coords = numpy.array(ext[u'coordinates'])
                    mins.append(numpy.min(coords, axis=1))
                    maxs.append(numpy.max(coords, axis=1))
                ext_min, ext_max = list(numpy.min(mins, axis=1)), list(numpy.max(maxs, axis=1))
                global_extent = [
                    [ext_min[0][1], ext_min[0][0]],
                    [ext_max[0][1], ext_max[0][0]]]

            content_type = 'text/html'
            results = j2_env.get_template('leaflet_map.html').render(title='Reference object: %s' % group_id, center='[%f, %f]' % (21.5, -102),
                                                                zoomlevel=5, geojson=ujson.dumps(results['geojson']),
                                                                     label_attribute='ref_name',
                                                                extent=ujson.dumps(global_extent))
        else:
            description = 'Unknown format given %s.' % (format)
            raise falcon.HTTPBadRequest('Reference', description,
                                        href='http://docs.example.com/auth')

        if can_zip_response(req.headers):
            resp.set_header('Content-Type', content_type)
            resp.set_header('Content-Encoding', 'gzip')
            if content_type == 'application/json':
                resp.body = compress_body(ujson.dumps(results))
            else:
                resp.body = compress_body(results)
        else:
            resp.set_header('Content-Type', content_type)
            if content_type == 'application/json':
                resp.body = ujson.dumps(results)
            else:
                resp.body = results
        resp.status = falcon.HTTP_200

class ReferenceSearcher():
    """
    http://localhost:8000/reference/search/count?entity_name=Mexico
    """

    # cors = cors

    def __init__(self, my_router):
        self.router = my_router
        self.logger = logging.getLogger('eoss.' + __name__)

        self.ref_objects = Persistance().get_selected_references(PRIORITY.keys())
        self.ref_groups = dict()
        for obj in Persistance().get_all_reference_types():
            self.ref_groups[obj.id] = obj

    def on_get(self, req, resp):
        """Handles GET requests"""

        BASE_URL = get_base_url(req.url)

        start_time = time.time()
        results = dict()
        results['entities'] = list()
        results['fuzzyness'] = 5
        results['total_count'] = len(self.ref_objects)
        results['Levenshtein_distance_threshold'] = THRES
        results['total_groups'] = len(self.ref_groups)
        results['searched_groups'] = len(PRIORITY.keys())

        if 'entity_name' not in req.params.keys():
            description = 'entity_name not specified in query '
            raise falcon.HTTPBadRequest('KeyError', description,
                                        href='http://docs.example.com/auth')

        entity_name = req.params['entity_name']
        logger.info('[GET] /reference/search/count?entity_name+%s' % (entity_name))
        entity_name_length = len(entity_name)
        counter = 0
        similarities = list()

        for ref_obj in self.ref_objects:
            if ref_obj[0][:entity_name_length].lower() == entity_name.lower():
                similarities.append((100 - PRIORITY[ref_obj.referencetype_id], ref_obj.ref_name, ref_obj.referencetype_id, ref_obj.ref_id))
            elif entity_name.lower() in ref_obj[0].lower():
                similarities.append((100 - PRIORITY[ref_obj.referencetype_id], ref_obj.ref_name, ref_obj.referencetype_id, ref_obj.ref_id))

        if len(similarities) < results['fuzzyness']:
            word_size = -1  # len(entity_name)
            for ref_obj in self.ref_objects:
                if ref_obj.ref_name not in [x[1] for x in similarities]:
                    sim = fuzz.partial_ratio(entity_name, ref_obj.ref_name[:word_size])
                    if sim > THRES:
                        similarities.append((sim - 1, ref_obj.ref_name, ref_obj.referencetype_id, ref_obj.ref_id))
                        counter += 1

        sorted_similarities = sorted(set(similarities), key=lambda tup: (-tup[0], tup[1]), reverse=True)
        sorted_similarities.reverse()

        for item in sorted_similarities:
            results['entities'].append({'entity_id': item[1], 'distance': item[0], 'reference_id': item[3],
                                        'resource_json': BASE_URL + self.router.reverse('reference_entity',
                                                                                group_id=self.ref_groups[item[2]].id,
                                                                                reference_id=item[3],
                                                                                format='json'),
                                        'resource_geojson': BASE_URL + self.router.reverse('reference_entity',
                                                                                group_id=self.ref_groups[item[2]].id,
                                                                                reference_id=item[3],
                                                                                format='geojson'),
                                        'resource_html': BASE_URL + self.router.reverse('reference_entity',
                                                                                group_id=self.ref_groups[item[2]].id,
                                                                                reference_id=item[3],
                                                                                format='html'),
                                        'entity_group': {
                                            'group_id': self.ref_groups[item[2]].id,
                                            'group_name': self.ref_groups[item[2]].name,
                                            'group_description': self.ref_groups[item[2]].description,
                                            'group_shortcut': self.ref_groups[item[2]].shortcut
                                        }})

        results['counter'] = counter
        results['processing_time'] = time.time() - start_time
        resp.status = falcon.HTTP_200

        if can_zip_response(req.headers):
            resp.set_header('Content-Type', 'application/json')
            resp.set_header('Content-Encoding', 'gzip')
            resp.body = compress_body(ujson.dumps(results))
        else:
            resp.set_header('Content-Type', 'application/json')
            resp.body = ujson.dumps(results)


text:
import numpy as np

from .helpers import SeededTest
from pymc3 import Model, Uniform, Normal, find_MAP, Slice, sample
from pymc3 import families, GLM, LinearComponent
import pandas as pd

# Generate data
def generate_data(intercept, slope, size=700):
    x = np.linspace(-1, 1, size)
    y = intercept + x * slope
    return x, y

class TestGLM(SeededTest):
    @classmethod
    def setup_class(cls):
        super(TestGLM, cls).setup_class()
        cls.intercept = 1
        cls.slope = 3
        cls.sd = .05
        x_linear, cls.y_linear = generate_data(cls.intercept, cls.slope, size=1000)
        cls.y_linear += np.random.normal(size=1000, scale=cls.sd)
        cls.data_linear = pd.DataFrame(dict(x=x_linear, y=cls.y_linear))

        x_logistic, y_logistic = generate_data(cls.intercept, cls.slope, size=3000)
        y_logistic = 1 / (1 + np.exp(-y_logistic))
        bern_trials = [np.random.binomial(1, i) for i in y_logistic]
        cls.data_logistic = dict(x=x_logistic, y=bern_trials)

    def test_linear_component(self):
        with Model() as model:
            lm = LinearComponent.from_formula('y ~ x', self.data_linear)
            sigma = Uniform('sigma', 0, 20)
            Normal('y_obs', mu=lm.y_est, sd=sigma, observed=self.y_linear)
            start = find_MAP(vars=[sigma])
            step = Slice(model.vars)
            trace = sample(500, step=step, start=start, progressbar=False, random_seed=self.random_seed)

            assert round(abs(np.mean(trace['Intercept'])-self.intercept), 1) == 0
            assert round(abs(np.mean(trace['x'])-self.slope), 1) == 0
            assert round(abs(np.mean(trace['sigma'])-self.sd), 1) == 0

    def test_glm(self):
        with Model() as model:
            GLM.from_formula('y ~ x', self.data_linear)
            step = Slice(model.vars)
            trace = sample(500, step, progressbar=False, random_seed=self.random_seed)

            assert round(abs(np.mean(trace['Intercept'])-self.intercept), 1) == 0
            assert round(abs(np.mean(trace['x'])-self.slope), 1) == 0
            assert round(abs(np.mean(trace['sd'])-self.sd), 1) == 0

    def test_glm_link_func(self):
        with Model() as model:
            GLM.from_formula('y ~ x', self.data_logistic,
                    family=families.Binomial(link=families.logit))
            step = Slice(model.vars)
            trace = sample(1000, step, progressbar=False, random_seed=self.random_seed)

            assert round(abs(np.mean(trace['Intercept'])-self.intercept), 1) == 0
            assert round(abs(np.mean(trace['x'])-self.slope), 1) == 0

    def test_more_than_one_glm_is_ok(self):
        with Model():
            GLM.from_formula('y ~ x', self.data_logistic,
                    family=families.Binomial(link=families.logit),
                    name='glm1')
            GLM.from_formula('y ~ x', self.data_logistic,
                    family=families.Binomial(link=families.logit),
                    name='glm2')

    def test_from_xy(self):
        with Model():
            GLM(self.data_logistic['x'],
                self.data_logistic['y'],
                family=families.Binomial(link=families.logit),
                name='glm1')

ext: py

reason: import numpy,from pymc3


text:
from sqlalchemy.types import TypeDecorator, LargeBinary

import numpy as np
import blosc

class Array(TypeDecorator):
    impl = LargeBinary

    def process_bind_param(self, value, dialect):
        if value is not None:
            # Serialization of numpy array
            return blosc.pack_array(np.array(value))
        else:
            return None

    def process_result_value(self, value, dialect):
        if value is not None:
            # Deserialization of numpy array
            return blosc.unpack_array(value)
        else:
            return None

max_stars_repo_name: brymer-meneses/GDetect

max_issues_repo_name: brymer-meneses/GDetect

max_forks_repo_name: brymer-meneses/GDetect

reason: import numpy


text:
# -*- coding: utf-8 -*-

import numpy as np

from pyfr.integrators.dual.pseudo.base import BaseDualPseudoIntegrator
from pyfr.mpiutil import get_comm_rank_root, get_mpi

class BaseDualPseudoController(BaseDualPseudoIntegrator):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        # Stats on the most recent step
        self.pseudostepinfo = []

    def convmon(self, i, minniters, dt_fac=1):
        if i >= minniters - 1:
            # Compute the normalised residual
            resid = self._resid(self._idxcurr, self._idxprev, dt_fac)

            self._update_pseudostepinfo(i + 1, resid)
            return all(r <= t for r, t in zip(resid, self._pseudo_residtol))
        else:
            self._update_pseudostepinfo(i + 1, None)
            return False

    def _resid(self, rcurr, rold, dt_fac):
        comm, rank, root = get_comm_rank_root()

        # Get a set of kernels to compute the residual
        rkerns = self._get_reduction_kerns(rcurr, rold, method='resid',
                                           norm=self._pseudo_norm)

        # Bind the dynmaic arguments
        for kern in rkerns:
            kern.bind(dt_fac)

        # Run the kernels
        self.backend.run_kernels(rkerns, wait=True)

        # Pseudo L2 norm
        if self._pseudo_norm == 'l2':
            # Reduce locally (element types) and globally (MPI ranks)
            res = np.array([sum(e) for e in zip(*[r.retval for r in rkerns])])
            comm.Allreduce(get_mpi('in_place'), res, op=get_mpi('sum'))

            # Normalise and return
            return tuple(np.sqrt(res / self._gndofs))
        # Uniform norm
        else:
            # Reduce locally (element types) and globally (MPI ranks)
            res = np.array([max(e) for e in zip(*[r.retval for r in rkerns])])
            comm.Allreduce(get_mpi('in_place'), res, op=get_mpi('max'))

            # Normalise and return
            return tuple(np.sqrt(res))

    def _update_pseudostepinfo(self, niters, resid):
        self.pseudostepinfo.append((self.ntotiters, niters, resid))

class DualNonePseudoController(BaseDualPseudoController):
    pseudo_controller_name = 'none'
    pseudo_controller_needs_lerrest = False

    def pseudo_advance(self, tcurr):
        self.tcurr = tcurr

        for i in range(self.maxniters):
            # Take the step
            self._idxcurr, self._idxprev = self.step(self.tcurr)

            # Convergence monitoring
            if self.convmon(i, self.minniters, self._dtau):
                break

class DualPIPseudoController(BaseDualPseudoController):
    pseudo_controller_name = 'local-pi'
    pseudo_controller_needs_lerrest = True

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        sect = 'solver-time-integrator'

        # Error norm
        self._norm = self.cfg.get(sect, 'errest-norm', 'l2')
        if self._norm not in {'l2', 'uniform'}:
            raise ValueError('Invalid error norm')

        tplargs = {'nvars': self.system.nvars}

        # Error tolerance
        tplargs['atol'] = self.cfg.getfloat(sect, 'atol')

        # PI control values
        sord = self.pseudo_stepper_order
        tplargs['expa'] = self.cfg.getfloat(sect, 'pi-alpha', 0.7) / sord
        tplargs['expb'] = self.cfg.getfloat(sect, 'pi-beta', 0.4) / sord

        # Constants
        tplargs['maxf'] = self.cfg.getfloat(sect, 'max-fact', 1.01)
        tplargs['minf'] = self.cfg.getfloat(sect, 'min-fact', 0.98)
        tplargs['saff'] = self.cfg.getfloat(sect, 'safety-fact', 0.8)
        tplargs['dtau_maxf'] = self.cfg.getfloat(sect, 'pseudo-dt-max-mult',
                                                 3.0)

        if not tplargs['minf'] < 1 <= tplargs['maxf']:
            raise ValueError('Invalid pseudo max-fact, min-fact')

        if tplargs['dtau_maxf'] < 1:
            raise ValueError('Invalid pseudo-dt-max-mult')

        # Limits for the local pseudo-time-step size
        tplargs['dtau_min'] = self._dtau
        tplargs['dtau_max'] = tplargs['dtau_maxf'] * self._dtau

        # Register a kernel to compute local error
        self.backend.pointwise.register(
            'pyfr.integrators.dual.pseudo.kernels.localerrest'
        )

        for ele, shape, dtaumat in zip(self.system.ele_map.values(),
                                       self.system.ele_shapes, self.dtau_upts):
            # Allocate storage for previous error
            err_prev = self.backend.matrix(shape, np.ones(shape),
                                           tags={'align'})

            # Append the error kernels to the list
            for i, err in enumerate(ele.scal_upts):
                self.pintgkernels['localerrest', i].append(
                    self.backend.kernel(
                        'localerrest', tplargs=tplargs,
                        dims=[ele.nupts, ele.neles], err=err,
                        errprev=err_prev, dtau_upts=dtaumat
                    )
                )

        self.backend.commit()

    def localerrest(self, errbank):
        self.backend.run_kernels(self.pintgkernels['localerrest', errbank])

    def pseudo_advance(self, tcurr):
        self.tcurr = tcurr

        for i in range(self.maxniters):
            # Take the step
            self._idxcurr, self._idxprev, self._idxerr = self.step(self.tcurr)
            self.localerrest(self._idxerr)

            if self.convmon(i, self.minniters):
                break


text:
import numpy as np
import pandas as pd
from pandas import Series, DataFrame
import matplotlib.pyplot as plt

def addlabels(x,y):
    for i in range(len(x)):
        plt.text(i,y[i],y[i],ha = 'center')

def Credits_stream(courses):
    # plt.savefig("mygraph.png")
    count_eng = 0
    count_pro = 0
    count_sci = 0
    count_hum = 0
    credit_eng = 0
    credit_pro = 0
    credit_sci = 0
    credit_hum = 0
    for i in range(len(courses[0])):
        # print (courses[2][i])
        if courses[2][i] == 'Engineering':
            count_eng=count_eng+1
            credit_eng=credit_eng+int(courses[3][i])
        elif courses[2][i] == 'Professional':
            count_pro=count_pro+1
            credit_pro=credit_pro+int(courses[3][i])
        elif courses[2][i] == 'Science':
            count_sci=count_sci+1
            credit_sci=credit_sci+int(courses[3][i])
        elif courses[2][i] == 'Humanities':
            count_hum=count_hum+1
            credit_hum=credit_hum+int(courses[3][i])
    data=[credit_eng,credit_pro,credit_sci,credit_hum]
    labels = ['Engineering', 'Professional', 'Science', 'Humanities']
    addlabels(labels, data)
    plt.xticks(range(len(data)), labels)
    plt.xlabel('Category')
    plt.ylabel('Credits')
    plt.title('title')
    plt.bar(range(len(data)), data)
    plt.show()

def gpa_graph(gpas):
    labels=[]
    for i in range(len(gpas)):
        # print(gpas[i])
        labels.append("Sem "+str(i))
    data = gpas
    addlabels(labels, data)
    plt.xticks(range(len(data)), labels)
    plt.xlabel('Semester')
    plt.ylabel('GPAs')
    plt.title('title')
    plt.bar(range(len(data)), data)
    plt.show()

max_stars_repo_name: Adil-MohammedK/viewgrades-scraper

max_issues_repo_name: Adil-MohammedK/viewgrades-scraper

max_forks_repo_name: Adil-MohammedK/viewgrades-scraper


text:
import numpy as np
def multiplication_table(row,col):
    return np.array([(x//col + 1)*(x%col + 1) for x in range(row*col)]).reshape(row, col).tolist()

max_stars_repo_name: opastushkov/codewars-solutions

max_issues_repo_name: opastushkov/codewars-solutions

max_forks_repo_name: opastushkov/codewars-solutions


text:
import math
import numpy

from .mulmom import MulMom

class T(list):
    """ Implements an interaction tensor T according Stones

        Elements are stored in canonical order for orders > 0, i.e.
        the order that one would expect: x, y, z for order = 1 or
        xx, xy, xz, yy, yz, zz for order = 2
    """
    def __init__(self, order, Rc, **kwargs):
        """ Creates an interaction tensor

            The interaction tensor is defined for a separation distance Rc
            to a specific order both given as arguments

            Arguments:
            order -- the order of the interaction tensor
            Rc -- the separation (or interaction) distance

            Keyword Arguments:
            verbose -- specifies verbosity. Accepted values are True or False
        """
        self.verbose = kwargs.get('verbose', False)
        if not isinstance(self.verbose, bool):
            raise ValueError("Keyword Argument 'verbose' has wrong type.")

        self.order = order
        tensor_sizes = {0: 1, 1: 3, 2: 6, 3: 10, 4: 15}
        if order < 0:
            raise ValueError("negative order not supported")
        if order > 5:
            raise NotImplementedError("order {} not implemented for interaction tensor".format(order))

        list.__init__(self, [0.0]*tensor_sizes[order])
        x = Rc[0]
        y = Rc[1]
        z = Rc[2]
        r = math.sqrt(x*x + y*y + z*z)

        if r < 1.0e-9:
            raise ValueError("Distance R is close to zero.")

        self._x = x
        self._y = y
        self._z = z

        if order == 0:
            # eq 3.1.4
            self[0] = 1 / r
            self._np = list(self)
        elif order == 1:
            # eq 3.1.5
            R3 = r**3
            self[0] = -x / R3
            self[1] = -y / R3
            self[2] = -z / R3
            self._np = list(self)
        elif order == 2:
            # eq 3.1.6
            R5 = r**5
            self[0] = (3*x*x - r**2) / R5 # xx
            self[1] = (3*x*y       ) / R5 # xy
            self[2] = (3*x*z       ) / R5 # xz
            self[3] = (3*y*y - r**2) / R5 # yy
            self[4] = (3*y*z       ) / R5 # xz
            self[5] = (3*z*z - r**2) / R5 # zz
            self._np = [[self[0], self[1], self[2]],
                        [self[1], self[3], self[4]],
                        [self[2], self[4], self[5]]]
        elif order == 3:
            # eq 3.1.7
            R7 = r**7
            self[0] = -(15*x*x*x - 3*r*r * (x + x + x))/ R7 # xxx
            self[1] = -(15*x*x*y - 3*r*r * y) / R7          # xxy
            self[2] = -(15*x*x*z - 3*r*r * z) / R7          # xxz
            self[3] = -(15*x*y*y - 3*r*r * x) / R7          # xyy
            self[4] = -(15*x*y*z) / R7                      # xyz
            self[5] = -(15*x*z*z - 3*r*r * x) / R7          # xzz
            self[6] = -(15*y*y*y - 3*r*r * (y + y + y))/ R7 # yyy
            self[7] = -(15*y*y*z - 3*r*r * z) / R7          # yyz
            self[8] = -(15*y*z*z - 3*r*r * y) / R7          # yzz
            self[9] = -(15*z*z*z - 3*r*r * (z + z + z))/ R7 # zzz
            self._np = None
        elif order == 4:
            # eq 3.1.8
            R9 = r**9
            R2 = r**2
            R4 = R2 * R2
            self[ 0] = (105 *x*x*x*x - 15* R2 * (x*x + x*x + x*x + x*x + x*x + x*x) + 3* R4 * (1 + 1 + 1)) / R9  # xxxx
            self[ 1] = (105 *x*x*x*y - 15* R2 * (  0 +   0 + x*y +   0 + x*y + x*y) + 3* R4 * (0 + 0 + 0)) / R9  # xxxy
            self[ 2] = (105 *x*x*x*z - 15* R2 * (  0 +   0 + x*z +   0 + x*z + x*z) + 3* R4 * (0 + 0 + 0)) / R9  # xxxz
            self[ 3] = (105 *x*x*y*y - 15* R2 * (x*x +   0 +   0 +   0 +   0 + y*y) + 3* R4 * (1 + 0 + 0)) / R9  # xxyy
            self[ 4] = (105 *x*x*y*z - 15* R2 * (  0 +   0 +   0 +   0 +   0 + y*z) + 3* R4 * (0 + 0 + 0)) / R9  # xxyz
            self[ 5] = (105 *x*x*z*z - 15* R2 * (x*x +   0 +   0 +   0 +   0 + z*z) + 3* R4 * (1 + 0 + 0)) / R9  # xxzz
            self[ 6] = (105 *x*y*y*y - 15* R2 * (x*y + x*y + x*y +   0 +   0 +   0) + 3* R4 * (0 + 0 + 0)) / R9  # xyyy
            self[ 7] = (105 *x*y*y*z - 15* R2 * (  0 +   0 + x*z +   0 +   0 +   0) + 3* R4 * (0 + 0 + 0)) / R9  # xyyz
            self[ 8] = (105 *x*y*z*z - 15* R2 * (x*y +   0 +   0 +   0 +   0 +   0) + 3* R4 * (0 + 0 + 0)) / R9  # xyzz
            self[ 9] = (105 *x*z*z*z - 15* R2 * (x*z + x*z + x*z +   0 +   0 +   0) + 3* R4 * (0 + 0 + 0)) / R9  # xzzz
            self[10] = (105 *y*y*y*y - 15* R2 * (y*y + y*y + y*y + y*y + y*y + y*y) + 3* R4 * (1 + 1 + 1)) / R9  # yyyy
            self[11] = (105 *y*y*y*z - 15* R2 * (  0 +   0 + y*z +   0 + y*z + y*z) + 3* R4 * (0 + 0 + 0)) / R9  # yyyz
            self[12] = (105 *y*y*z*z - 15* R2 * (y*y +   0 +   0 +   0 +   0 + z*z) + 3* R4 * (1 + 0 + 0)) / R9  # yyzz
            self[13] = (105 *y*z*z*z - 15* R2 * (y*z + y*z + y*z +   0 +   0 +   0) + 3* R4 * (0 + 0 + 0)) / R9  # yzzz
            self[14] = (105 *z*z*z*z - 15* R2 * (z*z + z*z + z*z + z*z + z*z + z*z) + 3* R4 * (1 + 1 + 1)) / R9  # zzzz
        else:
            raise NotImplementedError

    def __array__(self):
        if self._np is None:
            raise NotImplementedError
        return numpy.array(self._np)

    def __repr__(self):
        return "T({}, [{}, {}, {}])".format(self.order, self._x, self._y, self._z)

    def __mul__(self, other):
        if self.verbose:
            print("T.mul:", repr(self), "*", repr(other))
        if isinstance(other, MulMom):

            # potential of charge
            if self.order == 0 and other.order == 0:
                return MulMom(self[0] * other[0])

            # electric field of charge
            if self.order == 1 and other.order == 0:
                result = [0.0, 0.0, 0.0]
                try:
                    for i in range(3):
                        result[i] = self[i] * other[0]
                except TypeError:
                    print("other:", other[0], " type:", type(other[0]))
                    print("self :", self[i], " type: ", type(self[i]))
                return MulMom(*result)

            # electric field gradient of charge
            if self.order == 2 and other.order == 0:
                result = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
                for i in range(6):
                    result[i] = self[i] * other[0]
                return MulMom(*result)

            # T(0, R) * dipole is undefined
            if self.order == 0 and other.order == 1:
                raise ValueError("T(0, R) * dipole is undefined.")

            # potential of dipole
            if self.order == 1 and other.order == 1:
                result = sum([self[i] * other[i] for i in range(3)])
                return MulMom(result)

            # electric field of dipole
            if self.order == 2 and other.order == 1:
                result = [0.0, 0.0, 0.0]
                mux = other[0]
                muy = other[1]
                muz = other[2]
                #print("oo", list(other))
                #print("mu", mux, muy, muz)
                result[0] = self[0] * mux + self[1] * muy + self[2] * muz # xx * x + xy * y + xz * z
                result[1] = self[1] * mux + self[3] * muy + self[4] * muz # yx * x + yy * y + yz * z
                result[2] = self[2] * mux + self[4] * muy + self[5] * muz # zx * x + zy * y + zz * z
                return MulMom(*result)

            # electric field gradient of dipole
            if self.order == 3 and other.order == 1:
                result = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
                mux = other[0]
                muy = other[1]
                muz = other[2]
                result[0] = self[0] * mux + self[1] * muy + self[2] * muz
                result[1] = self[1] * mux + self[3] * muy + self[4] * muz
                result[2] = self[2] * mux + self[4] * muy + self[5] * muz
                result[3] = self[3] * mux + self[6] * muy + self[7] * muz
                result[4] = self[4] * mux + self[7] * muy + self[8] * muz
                result[5] = self[5] * mux + self[8] * muy + self[9] * muz
                return MulMom(*result)

            # T(0, R) * quadrupole is undefined
            if self.order == 0 and other.order == 2:
                raise ValueError("T(0, R) * quadrupole is undefined.")

            # T(1, R) * quadropole is undefined
            if self.order == 1 and other.order == 2:
                raise ValueError("T(1, R) * quadrupole is undefined.")

            # potential of quadrupole
            if self.order == 2 and other.order == 2:
                # first add diagonal parts xx, yy, zz
                result = self[0]*other[0] + self[3]*other[3] + self[5]*other[5]
                # then add off-diagonal parts (multiplied by two to include double counting) xy, xz, yz
                result += 2.0*(self[1]*other[1] + self[1]*other[1] + self[4]*other[4])
                return MulMom(result)

            # electric field of quadrupole
            if self.order == 3 and other.order == 2:
                result = [0.0, 0.0, 0.0]
                Oxx = other[0]
                Oxy = other[1]
                Oxz = other[2]
                Oyy = other[3]
                Oyz = other[4]
                Ozz = other[5]
                result[0] = self[0] * Oxx + self[1] * Oxy + self[2] * Oxz + self[ 3] * Oyy + self[ 4] * Oyz + self[ 5] * Ozz
                result[1] = self[1] * Oxx + self[3] * Oxy + self[4] * Oxz + self[ 6] * Oyy + self[ 7] * Oyz + self[ 8] * Ozz
                result[2] = self[2] * Oxx + self[4] * Oxy + self[5] * Oxz + self[ 7] * Oyy + self[ 8] * Oyz + self[ 9] * Ozz
                return MulMom(*result)

            # electric field gradient of quadrupole
            if self.order == 4 and other.order == 2:
                result = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
                Oxx = other[0]
                Oxy = other[1]
                Oxz = other[2]
                Oyy = other[3]
                Oyz = other[4]
                Ozz = other[5]
                result[0] = self[0] * Oxx + self[1] * Oxy + self[2] * Oxz + self[ 3] * Oyy + self[ 4] * Oyz + self[ 5] * Ozz
                result[1] = self[1] * Oxx + self[3] * Oxy + self[4] * Oxz + self[ 6] * Oyy + self[ 7] * Oyz + self[ 8] * Ozz
                result[2] = self[2] * Oxx + self[4] * Oxy + self[5] * Oxz + self[ 7] * Oyy + self[ 8] * Oyz + self[ 9] * Ozz
                result[3] = self[3] * Oxx + self[6] * Oxy + self[7] * Oxz + self[10] * Oyy + self[11] * Oyz + self[12] * Ozz
                result[4] = self[4] * Oxx + self[7] * Oxy + self[8] * Oxz + self[11] * Oyy + self[12] * Oyz + self[13] * Ozz
                result[5] = self[5] * Oxx + self[8] * Oxy + self[9] * Oxz + self[12] * Oyy + self[13] * Oyz + self[14] * Ozz
                return MulMom(*result)

            raise NotImplementedError("Your request of T({}, R) * multipole order {} is not implemented.".format(self.order, other.order))
        else:
            return TypeError("Expected {}".format(type(other)))

max_stars_repo_name: steinmanngroup/pepytools

max_issues_repo_name: steinmanngroup/pepytools

max_forks_repo_name: steinmanngroup/pepytools


text:
import numpy as np
import os
import datetime
import argparse
import pickle
from shutil import copyfile
import time

from sklearn.model_selection import StratifiedKFold

from geocoding import features_utilities as feat_ut, writers as wrtrs
from geocoding.config import Config

def main():
    """
    Implements the first step of the experiment pipeline. Creates feature \
    sets for each one of the folds.

    Returns:
        None
    """
    # Construct argument parser and parse arguments
    ap = argparse.ArgumentParser()
    ap.add_argument('-fpath', required=True)
    args = vars(ap.parse_args())

    t1 = time.time()

    # Create folder to store experiment
    date_time = datetime.datetime.now().strftime("%d-%m-%Y_%H-%M-%S")
    exp_path = os.path.join(Config.base_dir, 'experiments', f'exp_{date_time}')
    os.makedirs(exp_path)

    # Create folder to store feature extraction results
    results_path = os.path.join(exp_path, 'features_extraction_results')
    os.makedirs(results_path)
    copyfile('./geocoding/config.py', os.path.join(exp_path, 'config.py'))

    # Load dataset into dataframe
    df = feat_ut.load_points_df(os.path.join(Config.base_dir, args['fpath']))
    # Shuffle dataframe
    df = df.sample(frac=1).reset_index(drop=True)
    # Encode labels
    df, encoder = feat_ut.encode_labels(df)

    df.to_csv(os.path.join(results_path, 'train_df.csv'), index=False)
    pickle.dump(encoder, open(os.path.join(results_path, 'encoder.pkl'), 'wb'))

    feat_ut.get_required_external_files(df, results_path)

    addresses, targets = list(df['address']), list(df['target'])
    skf = StratifiedKFold(n_splits=Config.n_folds)
    fold = 1

    for train_idxs, test_idxs in skf.split(addresses, targets):
        fold_path = os.path.join(results_path, 'fold_' + str(fold))
        os.makedirs(fold_path)
        os.makedirs(os.path.join(fold_path, 'features'))
        os.makedirs(os.path.join(fold_path, 'pickled_objects'))

        X_train = feat_ut.create_train_features(df.iloc[train_idxs].reset_index(), results_path, fold_path)
        X_test = feat_ut.create_test_features(
            df.iloc[test_idxs].reset_index(), results_path, os.path.join(fold_path, 'pickled_objects'), fold_path
        )
        y_train, y_test = df['target'][train_idxs], df['target'][test_idxs]

        np.save(os.path.join(fold_path, f'X_train.npy'), X_train)
        np.save(os.path.join(fold_path, f'X_test.npy'), X_test)
        np.save(os.path.join(fold_path, f'y_train.npy'), y_train)
        np.save(os.path.join(fold_path, f'y_test.npy'), y_test)
        fold += 1

    wrtrs.write_feats_space(os.path.join(results_path, 'included_features.csv'))

    print(f'Feature extraction done in {time.time() - t1:.3f} sec.')
    return

if __name__ == "__main__":
    main()

max_stars_repo_name: LinkGeoML/Geocoding

max_issues_repo_name: LinkGeoML/Geocoding

max_forks_repo_name: LinkGeoML/Geocoding


text:
# ----- I M P O R T S ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- -----
import sys, getpass, traceback
import warnings

import pandas as pd
import numpy as np
# from sklearn.linear_model import LinearRegression

from datetime import datetime, timedelta

import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages

from dotenv import dotenv_values
from Client import Client # can be found in same dir as this file in repositorie

# ----- M E T A D A T A ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- -----

__version__ = 'v1.0.0'
__description__ = 'Script to fetch Pepper data and create .pdf containing weekly report'
__author__ = 'Benjamin Thomas Schwertfeger'
__copyright__ = 'Benjamin Thomas Schwertfeger'
__email__ = 'development@b-schwertfeger.de'
__status__ = 'Production'
__github__ = 'https://github.com/ProjectPepperHSB/Backend-Services.git'

# ----- S E T T I N G S ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- -----

if sys.platform == 'linux' or sys.platform == 'linux2':
    out_dir = f'/home/docker-hbv-kms/weekly-reports'
elif sys.platform == 'darwin':
    out_dir = f'/Users/{getpass.getuser()}/repositories/Backend-Services/analysis/out'
elif sys.platform == 'win32':
    exit() # enter your windows path path

warnings.filterwarnings('ignore')

plt.rcParams['figure.figsize'] = [10, 6]
plt.rcParams['savefig.bbox'] = 'tight'
plt.style.use('fivethirtyeight')

_red, _orange, _gray, _green, _blue = '#fc4f30', '#e5ae38', '#8b8b8b', '#6d904f', '#30a2da'
_weekdays = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']

# ----- ----- -----
LOOKBACK = 7 #days

now = datetime.now()
start_date = (now - timedelta(days=LOOKBACK)).strftime('%Y-%m-%d')
out_fname = f'weekly_report_{start_date}_-_{now.strftime("%Y-%m-%d")}.pdf'

# ----- S E T U P ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- -----

not_understand_df, emotion_states_df, use_case_df = None, None, None

# ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- -----

def init() -> None:
    global not_understand_df, emotion_states_df, use_case_df

    try:
        config = dotenv_values('.env')
        API_KEY = config['API_KEY']
    except KeyError:
        print('No .env file with API_KEY found!')
        exit()

    try:
        client = Client(API_KEY, sandbox=False, verbose=1) # create client to connect with web-app

        query_str = f'WHERE ts > NOW() - INTERVAL {LOOKBACK} day'

        not_understand_df = pd.DataFrame(data=client.sql_query(f'SELECT * FROM pepper_did_not_understand_table {query_str}'))
        emotion_states_df = pd.DataFrame(data=client.sql_query(f'SELECT * FROM pepper_emotion_table {query_str}'))
        use_case_df = pd.DataFrame(data=client.sql_query(f'SELECT * FROM pepper_use_case_table {query_str}'))

        # preprocessing
        emotion_states_df['dialog_time'] = np.array([x for x in emotion_states_df['dialog_time']]).astype('float32')
    except:
        print(f'Could not fetch data from backend!\n{traceback.format_exc()}')
        print('Check your internet connection and check if the backend service is running!')
        exit()

def plots(pdf) -> None:
    # PIE ----- USE-CASE-USAGE -----
    plt.figure()
    fig = use_case_df['use_case'].value_counts().plot(kind='pie', autopct='%1.1f%%').get_figure()
    plt.title('Use-Case usage'); plt.axis('off'); pdf.savefig(fig)

    # PIE ----- GENDER-DISTRIBUTION -----
    plt.figure()
    fig = emotion_states_df.gender.value_counts().plot(kind='pie', autopct='%1.1f%%',colors = [ _red, _blue ]).get_figure()
    plt.title('Gender distribution'); plt.axis('off'); pdf.savefig(fig)

    # PIE ----- BASIC-EMOTION -----
    plt.figure()
    fig = emotion_states_df['basic_emotion'].value_counts().plot(kind='pie', autopct='%1.1f%%',colors = [ _gray, _red, _orange, _green]).get_figure()
    plt.title('Distribution of basic emotion occurrence'); plt.axis('off'); pdf.savefig()

    # BARH ----- DISTRIBUTION OF EMOTION BY GENDER -----
    plt.figure()
    fig = pd.concat(
        [emotion_states_df[['gender', 'basic_emotion']].pivot_table(index=['gender'], columns=col, aggfunc=len) for col in ['basic_emotion']], axis = 1
    ).fillna(0).plot(kind = 'barh', color = { 'bad': _red, 'bored': _gray, 'excited': _orange, 'good': _green }).get_figure()
    plt.title('Distributions of basic emotions grouped by gender'); plt.xlabel('count'); pdf.savefig()

    # BARH ----- DISTRIBUTION OF PLEASURE STATES ------
    plt.figure()
    fig = pd.concat(
        [emotion_states_df[['gender', 'pleasure_state']].pivot_table(index=['gender'], columns=col, aggfunc=len) for col in ['pleasure_state']], axis=1
    ).fillna(0).plot(kind='barh', color={ 'bad': _red, 'medium': _gray, 'good': _orange, 'perfect': _green })
    plt.title('Distribution of pleasure states grouped by gender'); pdf.savefig()

    # ----- STATISTICS BY DAY OF WEEK ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ------ ----- ----- ----- -----
    d = emotion_states_df[['distance', 'gender', 'age', 'basic_emotion', 'pleasure_state', 'excitement_state', 'smile_state', 'dialog_time', 'ts']]
    d_use_case = use_case_df[['use_case', 'ts']]

    for i, ts in enumerate(d['ts']):
        date_obj = datetime.strptime(ts, '%Y-%m-%dT%H:%M:%S.000Z')
        d['ts'][i] = f'{date_obj.day}.{date_obj.month}.{date_obj.year} {date_obj.hour}:{date_obj.minute}'
    for i, ts in enumerate(d_use_case['ts']):
        date_obj = datetime.strptime(ts, '%Y-%m-%dT%H:%M:%S.000Z')
        d_use_case['ts'][i] = f'{date_obj.day}.{date_obj.month}.{date_obj.year} {date_obj.hour}:{date_obj.minute}'

    d['weekday'] = [datetime.strptime(ts, '%d.%m.%Y %H:%M').weekday() for ts in d['ts']]
    d_use_case['weekday'] = [datetime.strptime(ts, '%d.%m.%Y %H:%M').weekday() for ts in d_use_case['ts']]

    # BAR ----- EMOTIONS BY WEEKDAY
    plt.figure()
    fig = pd.concat(
        [d[['weekday', 'basic_emotion']].pivot_table(index=['weekday'], columns=col, aggfunc=len) for col in ['basic_emotion']], axis = 1
    ).fillna(0).plot(kind = 'bar', color = { 'bad': _red, 'bored': _gray, 'excited': _orange, 'good': _green }, rot = 35)

    plt.title('Distribution of basic emotions grouped by weekday')
    plt.xticks(np.arange(len(plt.gca().get_xticklabels())), [_weekdays[int(i.get_text())] for i in plt.gca().get_xticklabels()])
    plt.xlabel('weekday'); plt.ylabel('count'); pdf.savefig()

    # BAR ----- GENDER BY WEEKDAY
    plt.figure()
    fig = pd.concat(
        [d[['weekday', 'gender']].pivot_table(index=['weekday'], columns=col, aggfunc=len) for col in ['gender']], axis = 1
    ).fillna(0).plot(kind = 'bar', rot = 35)

    plt.title('Usage by gender and weekday')
    plt.xticks(np.arange(len(plt.gca().get_xticklabels())), [_weekdays[int(i.get_text())] for i in plt.gca().get_xticklabels()])
    plt.xlabel('weekday'); plt.ylabel('count'); pdf.savefig()

    # BAR ----- Dialog time by weekday
    plt.figure()
    fig = d[['weekday', 'dialog_time']].groupby('weekday').mean().plot(kind='bar', rot=35)
    plt.title('Mean dialog time by weekday')
    plt.xticks(np.arange(len(plt.gca().get_xticklabels())), [_weekdays[int(i.get_text())] for i in plt.gca().get_xticklabels()])
    plt.xlabel('weekday'); plt.ylabel('dialog time in minutes'); pdf.savefig()

    plt.figure()
    fig = pd.concat(
        [d_use_case[['weekday', 'use_case']].pivot_table(index=['weekday'], columns=col, aggfunc=len) for col in ['use_case']],axis = 1
    ).fillna(0).plot(kind = 'bar', rot = 35)

    plt.title('Use case by weekday')
    plt.xticks(np.arange(len(plt.gca().get_xticklabels())), [_weekdays[int(i.get_text())] for i in plt.gca().get_xticklabels()])
    plt.xlabel('weekday')
    plt.ylabel('count')
    pdf.savefig()

    # ----- ----- ------ ----- ----- ----- ----- ----- ----- ------ ----- ----- ----- ----- ----- ----- ------ ----- ----- ----- -----

    # SCATTER ----- LINEAR REGRESSION -----
    # maybe not representative enough for 7 day period

    # data = emotion_states_df[['distance', 'age', 'gender', 'basic_emotion', 'pleasure_state', 'excitement_state', 'smile_state', 'dialog_time']]
    # X = data.iloc[:, 1].values.reshape(-1, 1).astype('int') # age
    # Y = data.iloc[:, -1].values.reshape(-1, 1).astype('float32') # dialog_time
    # linear_regressor = LinearRegression(); linear_regressor.fit(X, Y)
    # Y_pred = linear_regressor.predict(X)

    # plt.figure()
    # fig = plt.scatter(X, Y)
    # plt.plot(X, Y_pred, color='red')
    # plt.xlabel('age'); plt.ylabel('dialog time')
    # plt.title('Linear regression on dialog time and age')
    # pdf.savefig()

def main() -> None:
    init()

    with PdfPages(f'{out_dir}/{out_fname}') as pdf:
        plots(pdf)

        # metadata
        d = pdf.infodict()
        d['Title'] = f'Weekly report {start_date} - {now}'
        d['Author'] = 'Team Pepper'
        # d['Subject'] = '....'
        d['Keywords'] = 'pepper robot matplotlib plots report'
        d['CreationDate'] = now.strftime('%Y-%m-%d')

if __name__ == '__main__':
    main()

max_stars_repo_name: ProjectPepperHSB/Backend-Services

max_issues_repo_name: ProjectPepperHSB/Backend-Services

max_forks_repo_name: ProjectPepperHSB/Backend-Services


text:
from keras.optimizers import Adam, SGD
from keras.layers import LSTM, Dense, BatchNormalization, Dropout, Conv2D, Flatten, Embedding
from keras.models import Sequential
from keras import backend as K
import numpy as np
import tensorflow as tf
from sklearn.utils import shuffle
import matplotlib.pyplot as plt
import wfdb
import scikitplot as splt
from time import time
from beatclassification.data_visualization import data_visualization
from tensorflow.python.tools import freeze_graph, optimize_for_inference_lib
from keras.callbacks import Callback, EarlyStopping, TensorBoard
from sklearn.metrics import f1_score, precision_score, recall_score
from beatclassification.Preprocessing import Preprocessing
from beatclassification.Evaluation import Evaluation
from keras.models import load_model

dv = data_visualization()
prep = Preprocessing()
eval = Evaluation()

# noinspection PyTypeChecker
class NN:

    def create_LSTM_model( self, X_train, X_val, Y_train, Y_val, aami_classes, callbacks, n_LSTM_layers, n_dense_layers,
                           batch_size,
                           activation, timesteps, n_neurons, optimizer, batch_normalization, epochs, dropout ):
        model = Sequential()
        input_shape = (timesteps, X_train.shape[ 2 ])
        if len(aami_classes) == 2:
            loss = 'binary_crossentropy'
            output_activation = 'sigmoid'
        else:
            loss = 'categorical_crossentropy'
            output_activation = 'softmax'
        if n_LSTM_layers > 1:
            model.add(
                LSTM(n_neurons, input_shape=input_shape, activation=activation, return_sequences=True))
            for i in range(n_LSTM_layers - 2):
                model.add(LSTM(n_neurons, activation=activation, return_sequences=True))
            model.add(LSTM(n_neurons, activation=activation))
        elif n_LSTM_layers == 1:
            input_shape = (timesteps, X_train.shape[ 2 ])
            model.add(
                LSTM(n_neurons, input_shape=input_shape, activation=activation))
        if n_dense_layers > 0 and n_LSTM_layers == 0:
            model.add(Dense(n_neurons, input_shape=(170,)))
            for i in range(n_dense_layers - 1):
                model.add(Dense(n_neurons, activation=activation))
                if batch_normalization:
                    model.add(BatchNormalization())
                if dropout is not None:
                    model.add(Dropout(dropout))
        elif n_dense_layers > 0:
            if batch_normalization:
                model.add(BatchNormalization())
            for i in range(n_dense_layers):
                model.add(Dense(n_neurons, activation=activation))
                if batch_normalization:
                    model.add(BatchNormalization())
                if dropout is not None:
                    model.add(Dropout(dropout))
        model.add(Dense(Y_train.shape[-1], activation=output_activation))
        optimizer = optimizer
        model.compile(loss=loss,
                      optimizer=optimizer)
        model.fit(X_train, Y_train, epochs=epochs, batch_size=batch_size, validation_data=(X_val, Y_val),
                  callbacks=callbacks, sample_weight=None)
        return model

    def create_CNN_model( self, X_train, X_val, Y_train, Y_val, aami_classes, callbacks, n_dense_layers, batch_size,
                          activation, width, n_neurons, optimizer, batch_normalization, epochs ):
        input_shape = (2, 170, 1)
        model = Sequential()
        model.add(Conv2D(n_neurons, (2, width), input_shape=input_shape, activation=activation))
        model.add(Flatten())
        if n_dense_layers > 0:
            if batch_normalization:
                model.add(BatchNormalization())
            for i in range(n_dense_layers):
                model.add(Dense(n_neurons, activation=activation))
                if batch_normalization:
                    model.add(BatchNormalization())
        model.add(Dense(len(aami_classes), activation='softmax'))
        optimizer = optimizer
        tensorboard = TensorBoard(log_dir="beatclassification/NN/logs/{}".format(time()), write_grads=True,
                                  histogram_freq=1)
        callbacks.append(tensorboard)
        model.compile(loss='categorical_crossentropy',
                      optimizer=optimizer)
        model.fit(X_train, Y_train, epochs=epochs, batch_size=batch_size, validation_data=(X_val, Y_val),
                  callbacks=callbacks)
        return model

    def beat_classification( self, classes=None, aami=True, augment=None, reduce=None, train_size=0.5, channels=None,
                             n_dense_layers=1, batch_size=32, standardize=True, multiclass=True, activation='relu',
                             timesteps=5, n_neurons=128, filtered=False, model=None, model_name='LSTM', epochs=100, train_db='mitdb',
                             dropout=0.5, vertical=True, n_LSTM_layers=1, window=None, left_window=70, right_window= 100,
                             callbacks=None, optimizer=None, batch_normalization=True, width=2, patience=10, validation=False):
        if classes is None:
            classes = ['N', 'S', 'V', 'F']
        if callbacks is None:
            callbacks = [ EarlyStopping(patience=patience, restore_best_weights=True),
                          TensorBoard(log_dir="beatclassification/NN/logs/{}".format(time()),
                                  histogram_freq=1)]
        if optimizer is None:
            optimizer = Adam()
        if channels is None:
            channels = [0]
        if vertical:
            X_train, Y_train, X_val, Y_val, X_test, Y_test = prep.vertical_split(classes=classes,
                                                                                timesteps=timesteps,
                                                                                channels=channels,
                                                                                standardize=standardize, aami=aami,
                                                                                model=model_name, train_size=train_size,
                                                                                multiclass=multiclass,
                                                                                window=window,
                                                                                left_window=left_window,
                                                                                right_window=right_window)
        # De Chazal horizontal split
        else:
            X_train, Y_train, X_val, Y_val, X_test, Y_test = prep.horizontal_split(classes=classes,
                                                                                timesteps=timesteps,
                                                                                channels=channels,
                                                                                standardize=standardize, aami=aami,
                                                                                model=model_name, train_db=train_db,
                                                                                multiclass=multiclass,
                                                                                window=window,
                                                                                left_window=left_window,
                                                                                right_window=right_window)
        dv.distribution(Y_train, classes=classes, multiclass=multiclass)
        if reduce is not None:
            for label in reduce:
                reduction_factor = reduce[ label ]
                if reduction_factor > 1:
                    X_train, Y_train = prep.subsample_data(X_train, Y_train, classes, label, reduction_factor,
                                                           one_hot=True)
        if augment is not None:
            for label in augment:
                print(label)
                augment_factor = augment[ label ]
                print(augment_factor)
                if augment_factor > 1:
                    X_train, Y_train = prep.augment_data(X_train, Y_train, classes, label, augment_factor,
                                                         one_hot=True)
        X_train, Y_train = shuffle(X_train, Y_train)
        print('train distribution')
        print(dv.distribution(Y_train, classes, multiclass=multiclass))
        print('val distribution')
        print(dv.distribution(Y_val, classes, multiclass=multiclass))
        print('test distribution')
        print(dv.distribution(Y_test, classes, multiclass=multiclass))
        if model is None:
            if model_name == 'LSTM':
                model = self.create_LSTM_model(X_train, X_val, Y_train, Y_val, classes, callbacks, n_LSTM_layers,
                                               n_dense_layers, batch_size,
                                               activation, timesteps, n_neurons, optimizer, batch_normalization, epochs,
                                               dropout)
            else:
                model = self.create_CNN_model(X_train, X_val, Y_train, Y_val, aami_classes=classes,
                                              n_dense_layers=n_dense_layers,
                                              batch_size=batch_size, activation=activation, n_neurons=n_neurons,
                                              batch_normalization=batch_normalization, callbacks=callbacks, width=width,
                                              optimizer=optimizer,
                                              epochs=epochs)
        if validation:
            predictions = model.predict(X_val, batch_size=32)
            return eval.evaluate(predictions, Y_val, plot=False), model
        else:
            predictions = model.predict(X_test, batch_size=32)
            eval.evaluate(predictions, Y_test, classes=classes,  plot=True, title='Confusion Matrix for LSTM network')
        #self.export_model(tf.train.Saver(), [ model_name.lower() + '_1_input' ], 'dense_2/Softmax', model_name)

    def export_model( self, saver, input_node_names, output_node_name, model_name ):
        tf.train.write_graph(K.get_session().graph_def, 'out', model_name + '_graph.pbtxt')
        saver.save(K.get_session(), 'out/' + model_name + '.chkp')
        freeze_graph.freeze_graph('out/' + model_name + '_graph.pbtxt', None,
                                  False, 'out/' + model_name + '.chkp', output_node_name,
                                  'save/restore_all', 'save/Const:0',
                                  'out/frozen_' + model_name + '.pb', True, '')
        input_graph_def = tf.GraphDef()
        with tf.gfile.Open('out/frozen_' + model_name + '.pb', 'rb') as f:
            input_graph_def.ParseFromString(f.read())
        output_graph_def = optimize_for_inference_lib.optimize_for_inference(input_graph_def, input_node_names,
                                                                             [ output_node_name ],
                                                                             tf.float32.as_datatype_enum)
        with tf.gfile.FastGFile('out/opt_' + model_name + '.pb', 'wb') as f:
            f.write(output_graph_def.SerializeToString())
        print('graph saved!')

if __name__ == '__main__':
    lstm = NN()
    augment = {'S':4, 'F':9}
    best_score = 0
    scores = list()
    best_window = None
    windows = [200, 300]
    for window in windows:
        score, model = lstm.beat_classification(window=window, augment=augment, validation=True, patience=10)
        scores.append(score)
        if score > best_score:
            best_score = score
            best_window = window
            model.save('best.h5')
        K.clear_session()
    print('best')
    print(best_window)
    print(best_score)
    model = load_model('best.h5')
    lstm.beat_classification(window=best_window, augment=augment, model=model)
    plt.close()
    plt.plot(windows, scores)
    plt.ylabel('F1 score')
    plt.xlabel('window size(samples)')
    plt.legend()
    plt.show()

max_issues_repo_name: jjongjjong/ECGClassification

max_forks_repo_name: jjongjjong/ECGClassification


text:
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
# Lint as: python3
"""Gradient-based attribution."""

import copy
import functools
from typing import Any, Iterable, List, Optional

from absl import logging
from lit_nlp.api import components as lit_components
from lit_nlp.api import dataset as lit_dataset
from lit_nlp.api import dtypes
from lit_nlp.api import model as lit_model
from lit_nlp.api import types
from lit_nlp.components.citrus import lime
from lit_nlp.components.citrus import utils as citrus_util
from lit_nlp.lib import utils

import numpy as np

JsonDict = types.JsonDict
Spec = types.Spec

TARGET_HEAD_KEY = 'Output field to explain'
CLASS_KEY = 'Class index to explain'
KERNEL_WIDTH_KEY = 'Kernel width'
MASK_KEY = 'Mask'
NUM_SAMPLES_KEY = 'Number of samples'
SEED_KEY = 'Seed'

def new_example(original_example: JsonDict, field: str, new_value: Any):
  """Deep copies the example and replaces `field` with `new_value`."""
  example = copy.deepcopy(original_example)
  example[field] = new_value
  return example

def _predict_fn(strings: Iterable[str], model: Any, original_example: JsonDict,
                text_key: str, pred_key: str, pred_type_info: types.LitType):
  """Given raw strings, return scores. Used by `lime.explain`.

  Adjust the `original_example` by changing the value of the field `text_key`
  by the values in `strings`, and run model prediction on each adjusted example,
  returning a list of scores, one entry per adjusted example.

  Args:
    strings: The adjusted strings to set in the original example.
    model: The model to run.
    original_example: The original example to adjust.
    text_key: The field in which to adjust the original example with the
      provided strings.
    pred_key: The key to the model's output field to explain.
    pred_type_info: The `LitType` value for the model's output field to explain.

  Returns:
    A list of scores for the model output on the adjusted examples. For
    regression tasks, a 1D list of values. For classification and multi-label,
    a 2D list, where each entry is a list of scores for each possible label,
    in order by the class index.
  """
  # Prepare example objects to be fed to the model for each sentence/string.
  input_examples = [new_example(original_example, text_key, s) for s in strings]

  # Get model predictions for the examples.
  model_outputs = model.predict(input_examples)
  outputs = [output[pred_key] for output in model_outputs]
  if isinstance(pred_type_info, types.SparseMultilabelPreds):
    assert pred_type_info.vocab, (
        f'No vocab found for {pred_key} field. Cannot use LIME.')
    # Convert list of class/score tuples to a list of scores for each possible
    # class, in class index order.
    output_arr = np.zeros([len(outputs), len(pred_type_info.vocab)])
    for i, pred in enumerate(outputs):
      for p in pred:
        class_idx = pred_type_info.vocab.index(p[0])
        output_arr[i, class_idx] = p[1]

    outputs = output_arr
  else:
    # Make outputs 1D in case of regression or binary classification.
    # <float32>[len(strings)] or <float32>[len(strings), num_labels].
    outputs = np.array(outputs)
    if outputs.ndim == 2 and outputs.shape[1] == 1:
      outputs = np.squeeze(outputs, axis=1)
  return outputs

def get_class_to_explain(provided_class_to_explain: int, model: Any,
                         pred_key: str, example: JsonDict) -> int:
  """Return the class index to explain.

  The provided class index can be -1, in which case this method determines
  which class to explain based on the class with the highest prediction score
  for the provided example.

  Args:
    provided_class_to_explain: The class index provided to this explainer.
    model: The model to run.
    pred_key: The key to the model's output field to explain.
    example: The example to explain.

  Returns:
    The true class index to explain, in the range [0, class vocab length).
  """
  pred_type_info = model.output_spec()[pred_key]
  # If provided_class_to_explain is -1, then explain the argmax class, for both
  # multiclass and sparse multilabel tasks.
  if ((isinstance(pred_type_info, types.MulticlassPreds) or
       isinstance(pred_type_info, types.SparseMultilabelPreds)) and
      provided_class_to_explain == -1):
    pred = list(model.predict([example]))[0][pred_key]
    if isinstance(pred_type_info, types.MulticlassPreds):
      return np.argmax(pred)
    else:
      # For sparse multi-label, sort class/score tuples to find the
      # highest-scoring class and get its vocab index.
      pred.sort(key=lambda elem: elem[1], reverse=True)
      class_name_to_explain = pred[0][0]
      return pred_type_info.vocab.index(class_name_to_explain)
  else:
    return provided_class_to_explain

class LIME(lit_components.Interpreter):
  """Local Interpretable Model-agnostic Explanations (LIME)."""

  def __init__(self):
    pass

  def run(
      self,
      inputs: List[JsonDict],
      model: lit_model.Model,
      dataset: lit_dataset.Dataset,
      model_outputs: Optional[List[JsonDict]] = None,
      config: Optional[JsonDict] = None,
  ) -> Optional[List[JsonDict]]:
    """Run this component, given a model and input(s)."""
    config_defaults = {k: v.default for k, v in self.config_spec().items()}
    config = dict(config_defaults, **(config or {}))  # update and return

    provided_class_to_explain = int(config[CLASS_KEY])
    kernel_width = int(config[KERNEL_WIDTH_KEY])
    num_samples = int(config[NUM_SAMPLES_KEY])
    mask_string = (config[MASK_KEY])
    # pylint: disable=g-explicit-bool-comparison
    seed = int(config[SEED_KEY]) if config[SEED_KEY] != '' else None
    # pylint: enable=g-explicit-bool-comparison

    # Find keys of input (text) segments to explain.
    # Search in the input spec, since it's only useful to look at ones that are
    # used by the model.
    text_keys = utils.find_spec_keys(model.input_spec(), types.TextSegment)
    if not text_keys:
      logging.warning('LIME requires text inputs.')
      return None
    logging.info('Found text fields for LIME attribution: %s', str(text_keys))

    # Find the key of output probabilities field(s).
    pred_keys = utils.find_spec_keys(
        model.output_spec(),
        (types.MulticlassPreds, types.RegressionScore,
         types.SparseMultilabelPreds))
    if not pred_keys:
      logging.warning('LIME did not find any supported output fields.')
      return None
    pred_key = config[TARGET_HEAD_KEY] or pred_keys[0]

    all_results = []

    # Explain each input.
    for input_ in inputs:
      # Dict[field name -> interpretations]
      result = {}
      predict_fn = functools.partial(
          _predict_fn, model=model, original_example=input_, pred_key=pred_key,
          pred_type_info=model.output_spec()[pred_key])

      class_to_explain = get_class_to_explain(
          provided_class_to_explain, model, pred_key, input_)

      # Explain each text segment in the input, keeping the others constant.
      for text_key in text_keys:
        input_string = input_[text_key]
        if not input_string:
          logging.info('Could not explain empty string for %s', text_key)
          continue
        logging.info('Explaining: %s', input_string)

        # Perturbs the input string, gets model predictions, fits linear model.
        explanation = lime.explain(
            sentence=input_string,
            predict_fn=functools.partial(predict_fn, text_key=text_key),
            # `class_to_explain` is ignored when predict_fn output is a scalar.
            class_to_explain=class_to_explain,
            num_samples=num_samples,
            tokenizer=str.split,
            mask_token=mask_string,
            kernel=functools.partial(
                lime.exponential_kernel, kernel_width=kernel_width),
            seed=seed)

        # Turn the LIME explanation into a list following original word order.
        scores = explanation.feature_importance
        # TODO(lit-dev): Move score normalization to the UI.
        scores = citrus_util.normalize_scores(scores)
        result[text_key] = dtypes.TokenSalience(input_string.split(), scores)

      all_results.append(result)

    return all_results

  def config_spec(self) -> types.Spec:
    matcher_types = ['MulticlassPreds', 'SparseMultilabelPreds',
                     'RegressionScore']
    return {
        TARGET_HEAD_KEY: types.FieldMatcher(spec='output', types=matcher_types),
        CLASS_KEY: types.TextSegment(default='-1'),
        MASK_KEY: types.TextSegment(default='[MASK]'),
        KERNEL_WIDTH_KEY: types.TextSegment(default='256'),
        NUM_SAMPLES_KEY: types.TextSegment(default='256'),
        SEED_KEY: types.TextSegment(default=''),
    }

  def is_compatible(self, model: lit_model.Model):
    text_keys = utils.find_spec_keys(model.input_spec(), types.TextSegment)
    pred_keys = utils.find_spec_keys(
        model.output_spec(),
        (types.MulticlassPreds, types.RegressionScore,
         types.SparseMultilabelPreds))
    return len(text_keys) and len(pred_keys)

  def meta_spec(self) -> types.Spec:
    return {'saliency': types.TokenSalience(autorun=False, signed=True)}

max_stars_repo_name: eichinflo/lit


text:
import cv2
import numpy as np
import json
import math
import shutil
import os

from lib.utils.keypoints_constants import part_mapping, advanced_parts, keypoint_names

def get_image_mean_color(img):
    return img.mean(axis=0).mean(axis=0)

def get_xy(center, widths, img_dims):
    x, y = center
    width_x, width_y = widths
    h, w = img_dims
    y1 = int(np.clip(y - width_y / 2, 0, h))
    y2 = int(np.clip(y + width_y / 2, 0, h))
    x1 = int(np.clip(x - width_x / 2, 0, w))
    x2 = int(np.clip(x + width_x / 2, 0, w))
    return x1, x2, y1, y2

def rect_cutout(image, x_center, y_center, x_length, y_length, color):
    x1, x2, y1, y2 = get_xy((x_center, y_center), (x_length, y_length), image.shape[:-1])
    image[y1:y2, x1:x2] = color
    return image

def circle_cutout(image, x_center, y_center, radius, color):
    h, w = image.shape[:-1]
    if x_center >= 0 and x_center < image.shape[0] and y_center >= 0 and y_center < image.shape[1]:
        x_center = max(x_center, 0)
        y_center = max(y_center, 0)
        x, y = np.ogrid[-y_center:h - y_center, -x_center:w - x_center]
        mask = x * x + y * y <= radius * radius

        image[mask] = color
    return image

def circle_blur(image, x_center, y_center, radius):
    # 9 times std (as it is 3 times
    blurred_img = cv2.GaussianBlur(image, (2*radius-1, 2*radius-1), 0)
    mask = np.zeros(image.shape, dtype=np.uint8)
    mask = circle_cutout(mask, x_center, y_center, radius, np.array([255, 255, 255]))
    out = np.where(mask == np.array([255, 255, 255]), blurred_img, image)

    return out

def blur(image, x_center, y_center, x_length, y_length):
    blurred_img = cv2.GaussianBlur(image, (31, 31), 0)
    mask = np.zeros(image.shape, dtype=np.uint8)
    mask = rect_cutout(mask, x_center, y_center, x_length, y_length, [255, 255, 255])
    out = np.where(mask == np.array([255, 255, 255]), blurred_img, image)

    return out

def get_image_dict(annotations):
    images_list = [(elem['id'], elem['file_name']) for elem in annotations['images']]
    return dict(images_list)

def extract_json(annotations, dataset):
    keypoints_dict = {}
    if dataset == "coco":
        images_dict = get_image_dict(annotations)
        for elem in annotations['annotations']:
            file_name = images_dict[elem['image_id']]
            if file_name not in keypoints_dict:
                keypoints_dict[file_name] = []
            keypoints = extract_keypoints(elem['keypoints'], dataset)
            if keypoints != {}:
                keypoints_dict[file_name].append(keypoints)
    else:
        for elem in annotations:
            if elem['image'] not in keypoints_dict:
                keypoints_dict[elem['image']] = []

            keypoints = extract_keypoints(elem['joints'], "mpii")
            if keypoints != {}:
                keypoints_dict[elem['image']].append(keypoints)

    return keypoints_dict

def extract_keypoints(keypoints_json, dataset):
    keypoints_pos = {}
    if dataset == "coco":
        for keypoint_id in range(len(keypoints_json) // 3):
            if keypoints_json[keypoint_id * 3 + 2] != 0:
                keypoints_pos[keypoint_names[dataset][keypoint_id]] = (
                    keypoints_json[keypoint_id * 3], keypoints_json[keypoint_id * 3 + 1])
    else:
        keypoints_pos = {}
        for i, elem in enumerate(keypoints_json):
            keypoints_pos[keypoint_names[dataset][i]] = (elem[0], elem[1])

    return keypoints_pos

def get_centerpoint(keypoints, key1, key2):
    keypoint1 = np.array(keypoints[key1])
    keypoint2 = np.array(keypoints[key2])
    return np.mean(np.array([keypoint1, keypoint2]), axis=0)

def get_shifted_point(keypoints, key1, key2, ratio):
    keypoint1 = np.array(keypoints[key1])
    keypoint2 = np.array(keypoints[key2])
    dist = ratio * abs(keypoint1 - keypoint2)
    return keypoint2 - dist

def get_dist(keypoints, key1, key2, key3=None):
    keypoint1 = np.array(keypoints[key1])
    keypoint2 = np.array(keypoints[key2])
    dist = abs(keypoint1 - keypoint2)
    min_dist = np.flip(0.25 * dist)
    dist = np.max([dist, min_dist], axis=0)
    if key3 and key3 in keypoints.keys():
        keypoint3 = np.array(keypoints[key3])
        dist2 = abs(keypoint2 - keypoint3)
        dist = np.array([dist, dist2]).max(axis=0)
    return dist

def calculate_center_with_dist(keypoints, chosen_keypoints):
    chosen_keypoints_pos = []
    for key in chosen_keypoints:
        if key in keypoints:
            chosen_keypoints_pos.append(keypoints[key])
    if len(chosen_keypoints_pos) > 1:
        chosen_keypoints_arr = np.array(chosen_keypoints_pos)
        min_pos = np.min(chosen_keypoints_arr, axis=0)
        max_pos = np.max(chosen_keypoints_arr, axis=0)
        dists = (max_pos - min_pos)
        center_point = np.mean(np.array([max_pos, min_pos]), axis=0)

        return center_point, dists
    return (0, 0), (0, 0)

def get_cords_with_width(keypoints, part, dataset):
    """
    Method using various heuristics to return coordinates of middle point for box
    :param img: image on which
    :param keypoints: dictionary of positions of various keypoints
    :param part: part which will be occluded

    :return: coordinates of a center point of box, width in X and Y of a box
    """
    if part == "head":
        if dataset == "coco":
            out = calculate_center_with_dist(keypoints, advanced_parts[dataset]['head'])
            return out[0], (1.2 * out[1][0], 1.2 * out[1][1])
        else:
            if "head_top" in keypoints.keys() and "upper_neck" in keypoints.keys():
                dists = get_dist(keypoints, "head_top", "upper_neck")
                return get_centerpoint(keypoints, "head_top", "upper_neck"), (2*dists[0], dists[1])
    elif part == "hip":
        if "left_hip" in keypoints.keys() and "right_hip" in keypoints.keys():
            return get_centerpoint(keypoints, "left_hip", "right_hip"), 2 * get_dist(keypoints, "left_hip",
                                                                                "right_hip")
    elif part == "left_arm":
        if "left_wrist" in keypoints.keys() and "left_elbow" in keypoints.keys() and "left_shoulder" in keypoints.keys():
            return calculate_center_with_dist(keypoints, ["left_wrist", "left_elbow", "left_shoulder"])
        elif "left_wrist" in keypoints.keys() and "left_elbow" in keypoints.keys():
            return keypoints['left_elbow'], 2 * get_dist(keypoints, "left_wrist", "left_elbow", "left_shoulder")

        elif "left_shoulder" in keypoints.keys() and "left_elbow" in keypoints.keys():
            return keypoints['left_elbow'], 2 * get_dist(keypoints, "left_shoulder", "left_elbow", "left_wrist")

        elif "left_shoulder" in keypoints.keys() and "left_wrist" in keypoints.keys():
            return get_centerpoint(keypoints, "left_shoulder", "left_wrist"), get_dist(keypoints, "left_wrist",
                                                                                "left_shoulder")
    elif part == "right_arm":
        if "right_wrist" in keypoints.keys() and "right_elbow" in keypoints.keys() and "right_shoulder" in keypoints.keys():
            return calculate_center_with_dist(keypoints, ["right_wrist", "right_elbow", "right_shoulder"])
        elif "right_wrist" in keypoints.keys() and "right_elbow" in keypoints.keys():
            return keypoints['right_elbow'], 2 * get_dist(keypoints, "right_wrist", "right_elbow", "right_shoulder")

        elif "right_shoulder" in keypoints.keys() and "right_elbow" in keypoints.keys():
            return keypoints['right_elbow'], 2 * get_dist(keypoints, "right_shoulder", "right_elbow")

        elif "right_shoulder" in keypoints.keys() and "right_wrist" in keypoints.keys():
            return get_centerpoint(keypoints, "right_shoulder", "right_wrist"), get_dist(keypoints, "right_wrist",
                                                                                "right_shoulder")
    elif part == "left_leg":
        if "left_hip" in keypoints.keys() and "left_knee" in keypoints.keys() and "left_ankle" in keypoints.keys():
            return calculate_center_with_dist(keypoints, ["left_ankle", "left_knee", "left_hip"])

        elif "left_hip" in keypoints.keys() and "left_knee" in keypoints.keys():
            return keypoints['left_knee'], 2 * get_dist(keypoints, "left_hip", "left_knee", "left_ankle")

        elif "left_ankle" in keypoints.keys() and "left_knee" in keypoints.keys():
            return keypoints['left_knee'], 2 * get_dist(keypoints, "left_ankle", "left_knee")

        elif "left_ankle" in keypoints.keys() and "left_hip" in keypoints.keys():
            return get_centerpoint(keypoints, "left_ankle", "left_hip"), get_dist(keypoints, "left_ankle", "left_hip")

    elif part == "right_leg":
        if "right_hip" in keypoints.keys() and "right_knee" in keypoints.keys() and "right_ankle" in keypoints.keys():
            return calculate_center_with_dist(keypoints, ["right_ankle", "right_knee", "right_hip"])

        elif "right_hip" in keypoints.keys() and "right_knee" in keypoints.keys():
            return keypoints['right_knee'], 2 * get_dist(keypoints, "right_hip", "right_knee", "right_ankle")

        elif "right_ankle" in keypoints.keys() and "right_knee" in keypoints.keys():
            return keypoints['right_knee'], 2 * get_dist(keypoints, "right_ankle", "right_knee", "right_ankle")

        elif "right_ankle" in keypoints.keys() and "right_hip" in keypoints.keys():
            return get_centerpoint(keypoints, "right_ankle", "right_hip"), get_dist(keypoints, "right_ankle",
                                                                                "right_hip", "right_knee")
    elif part == "corpus":
        return calculate_center_with_dist(keypoints, advanced_parts[dataset]["corpus"])
    elif part == "upper_body":
        return calculate_center_with_dist(keypoints, advanced_parts[dataset]["upper_body"])
    elif part == "lower_body":
        return calculate_center_with_dist(keypoints, advanced_parts[dataset]["lower_body"])
    elif part == "right_side":
        return calculate_center_with_dist(keypoints, advanced_parts[dataset]["right_side"])
    elif part == "left_side":
        return calculate_center_with_dist(keypoints, advanced_parts[dataset]["left_side"])

    return None  # Fallback option

def blurr_img(img, keypoints_list, part, dataset, width=6):
    concetrated_cutout = part not in part_mapping[dataset].keys()

    for keypoints in keypoints_list:
        if concetrated_cutout:
            if part in keypoints:
                # width = 6  # As it is in the ground truth
                x, y = keypoints[part]

                img = circle_blur(img, x, y, width)

                return img, (x, y), (width, width)
        else:
            cords = get_cords_with_width(keypoints, part, dataset)
            if cords:
                (x, y), (width_x, width_y) = cords

                img = blur(img, x, y, width_x, width_y)

                return img, (x, y), (width_x, width_y)
    return img, (0, 0), (0, 0)

def cutout_img(img, keypoints_list, part, mean_coloring, dataset, width = 6):
    color = [0, 0, 0]
    concetrated_cutout = part not in part_mapping[dataset].keys()
    if mean_coloring:
        color = get_image_mean_color(img)

    for keypoints in keypoints_list:
        if concetrated_cutout:
            if part in keypoints:
                x, y = keypoints[part]
                img = circle_cutout(img, x, y, width, color)
                return img, (x, y), (width, width)
        else:
            cords = get_cords_with_width(keypoints, part, dataset)
            if cords:
                (x, y), (width_x, width_y) = cords

                img = rect_cutout(img, x, y, width_x, width_y, color)

                return img, (x, y), (width_x, width_y)
    return img, (0, 0), (0, 0)

def copy_image_files(path, dst):
    if not os.path.exists(dst):
        os.mkdir(dst)
    for file in os.listdir(path):
        if file.endswith(".jpg"):
            shutil.copyfile(path + file, dst + file)

def remove_annotation(keypoint_arr, joints_vis, center_pos, widths):
    for id in range(keypoint_arr.shape[0]):
        pos = (keypoint_arr[id][0], keypoint_arr[id][1])
        if is_within_box(pos, center_pos, widths):
            keypoint_arr[id] = [0, 0, 0]
            joints_vis[id] = [0, 0, 0]

    return keypoint_arr, joints_vis

def is_within_box(pos, center_pos, widths):
    x_lower = center_pos[0] - widths[0]
    x_upper = center_pos[0] + widths[0]
    y_lower = center_pos[1] - widths[1]
    y_upper = center_pos[1] + widths[1]
    if widths[0] == 0 or widths[0] == 0:
        return False
    return x_lower < pos[0] <= x_upper and y_lower <= pos[1] <= y_upper


text:
from scipy import stats, special
import tasklogger
import pandas as pd
import numpy as np
import os
import sys

from . import utils

class SCYN:

    def __init__(self, seq='single-end', bin_len=500, ref='hg19', reg='*.bam', mapq=40, K=None, verbose=1):
        self.seq = seq
        self.bin_len = bin_len
        self.ref = ref
        self.reg = reg
        self.mapq = mapq
        self.verbose = verbose
        self._check_params()
        self.cnv = None
        self.meta_info = None
        self.segments = None
        self.bin_info = None
        self.K = K
        tasklogger.set_level(verbose)

    def _check_params(self):
        """Check SCYN parameters

        This allows us to fail early - otherwise certain unacceptable
        parameter choices, such as n='10.5', would only fail after
        minutes of runtime.

        Raises
        ------
        ValueError : unacceptable choice of parameters
        """
        utils.check_in(['single-end', 'paired-end'], seq=self.seq)
        utils.check_int(bin_len=self.bin_len)
        utils.check_positive(bin_len=self.bin_len)
        utils.check_in(['hg19', 'hg38'], ref=self.ref)
        utils.check_int(mapq=self.mapq)
        utils.check_positive(mapq=self.mapq)

    def set_params(self, **params):
        """Set the parameters of SCYN.

        Any parameters not given as named arguments will be left at their
        current value.

        Parameters
        ----------

        seq : string, optional, default: single-end
            The reads type: single-end or paired-end

        bin_len : int, optional, default: 500
            The bin length, default is 500K

        ref : string, optional, default: hg19
            The reference genome version: hg19 or hg38

        reg : string, optional, default: *.bam
            The regular expression to match all BAM files in your input directory.
            For example, "*.bam" will match all BAM files ended with '.bam'

        mapq : int, optional, default: 40
            The mapping quality cutoff when calculating the reads coverage

        K : int, optional, default: 10
            The predifined changepoints number for all chromosomes

        verbose : `int` or `boolean`, optional, default: 1
            If `True` or `> 0`, print log messages

        Returns
        -------
        self
        """

        # parameters
        if 'seq' in params and params['seq'] != self.seq:
            self.seq = params['seq']
            del params['seq']
        if 'bin_len' in params and params['bin_len'] != self.bin_len:
            self.bin_len = params['bin_len']
            del params['bin_len']
        if 'ref' in params and params['ref'] != self.ref:
            self.ref = params['ref']
            del params['ref']
        if 'reg' in params and params['reg'] != self.reg:
            self.reg = params['reg']
            del params['reg']
        if 'mapq' in params and params['mapq'] != self.mapq:
            self.mapq = params['mapq']
            del params['mapq']
        if 'verbose' in params:
            self.verbose = params['verbose']
            tasklogger.set_level(self.verbose)
            del params['verbose']

        self._check_params()
        return self

    def _cal_cnv(self, ref, Y, Ynor):
        all_break_points = dict.fromkeys(np.unique(ref['seqnames']))
        all_cnv = pd.DataFrame(columns=Y.columns)
        keys = list(all_break_points.keys())
        if 'chr' in keys[0]:
            keys = sorted(list(keys), key=lambda x: int(x[3:]))
        for chrom in keys:
            indexes = np.where(ref['seqnames'] == chrom)
            # tasklogger.log_info('Calculate CNV for chromosome ' + chrom+'...')
            breaks, cnv = self._cal_cnv_for_each_chrom(Y.iloc[indexes], Ynor.iloc[indexes])
            all_break_points[chrom] = breaks
            all_cnv = all_cnv.append(cnv)
        self.cnv = all_cnv
        self.segments = pd.DataFrame(columns=['Chromosome', 'Start_bin_no', 'End_bin_no'])
        for chrom in keys:
            break_points = all_break_points[chrom]
            for i in range(len(break_points) - 1):
                start = break_points[i]
                end = break_points[i+1] -1
                self.segments = self.segments.append([{
                    'Chromosome': chrom,
                    'Start_bin_no': start,
                    'End_bin_no': end
                }], ignore_index=True)
        return self

    def _get_segments(self, break_points):
        segments = np.zeros((len(break_points) -1, 2))
        for i in range(len(break_points) - 1):
            start = break_points[i]
            end = break_points[i+1] - 1
            segments[i] = [start, end]
        return segments

    def _cal_mbic(self, Y, nor_Y, segs):
        nor_Y.columns = Y.columns
        nor_Y.index = Y.index
        Y0 = Y.copy()
        nor_Y0 = nor_Y.copy()
        index_Y = np.column_stack(np.where(Y0 <= 20))
        for index in index_Y:
            Y0.iat[index[0], index[1]] = 20
        index_nor_Y = np.column_stack(np.where(nor_Y0 <= 20))
        for index in index_nor_Y:
            nor_Y0.iat[index[0], index[1]] = 20
        bin_num = Y.shape[0]
        sample_num = Y.shape[1]

        avg_mat = np.zeros((segs.shape[0], sample_num))
        rate_mat = np.zeros((segs.shape[0], sample_num))
        sum_Y_mat = np.zeros((segs.shape[0], sample_num))
        sum_nor_Y_mat = np.zeros((segs.shape[0], sample_num))
        sum_Y_mat0 = np.zeros((segs.shape[0], sample_num))
        sum_nor_Y_mat0 = np.zeros((segs.shape[0], sample_num))
        cumsum_Y = np.cumsum(Y).values
        cumsum_nor_Y = np.cumsum(nor_Y).values
        cumsum_Y0 = np.cumsum(Y0).values
        cumsum_nor_Y0 = np.cumsum(nor_Y0).values
        for index in range(segs.shape[0]):
            row = segs[index]
            start = int(row[0])
            end = int(row[1])
            if start == 0:
                sum_Y_mat[index] = cumsum_Y[end]
                sum_nor_Y_mat[index] = cumsum_nor_Y[end]
                sum_Y_mat0[index] = cumsum_Y0[end]
                sum_nor_Y_mat0[index] = cumsum_nor_Y0[end]
            else:
                sum_Y_mat[index] = cumsum_Y[end] - cumsum_Y[start - 1]
                sum_nor_Y_mat[index] = cumsum_nor_Y[end] - cumsum_nor_Y[start - 1]
                sum_Y_mat0[index] = cumsum_Y0[end] - cumsum_Y0[start - 1]
                sum_nor_Y_mat0[index] = cumsum_nor_Y0[end] - cumsum_nor_Y0[start - 1]
            avg_mat[index] = np.round(sum_Y_mat0[index]/(end-start+1))
            # print(end-start+1)
            rate_mat[index] = np.round(sum_Y_mat0[index]/sum_nor_Y_mat0[index] * 2)
        k1 = 3/2
        k2 = 2.27
        i = segs.shape[0] - 1
        # print(avg_mat.shape, rate_mat.shape, sum_Y_mat.shape, sum_nor_Y_mat.shape)
        rate_carriers = rate_mat[1:] - rate_mat[:i]
        rate_carriers[rate_carriers != 0] = 1
        avg_carriers = avg_mat[1:] - avg_mat[:i]
        total_carriers = avg_carriers * rate_carriers
        max_sum = np.max(
            np.sum(np.power(total_carriers, 2), axis=1))
        M = np.sum(rate_carriers)
        pi = M / (sample_num * (i))
        loglikeij = 0.0

        for row_index in np.arange(sum_Y_mat.shape[0]):
            row_sum_Y = sum_Y_mat[row_index]
            row_sum_nor_Y = sum_nor_Y_mat[row_index]
            row_sum_Y[row_sum_nor_Y < 20] = 20
            row_sum_nor_Y[row_sum_nor_Y < 20] = 20
            loglikeij += np.sum((1 - np.round(row_sum_Y / row_sum_nor_Y * 2) / 2) * row_sum_nor_Y
                                + np.log((np.round(row_sum_Y / row_sum_nor_Y * 2) + 1e-04) / 2.0001) * row_sum_Y)
        term1 = loglikeij
        # print(sys.argv[3], term1)
        if M == 0:
            term2 = 0
        else:
            term2 = -M / 2 * np.log(2 * loglikeij/M)
        term3 = -np.log(special.binom(bin_num, i))
        term4 = -M / 2
        if M == 0 or max_sum == 0:
            term5 = 0
        else:
            term5 = -np.sum(np.log(max_sum))
        term6 = -(i) * (k1 - k2)
        # print(pi)
        if pi == 0 or pi == 1:
            term7 = 0
        else:
            term7 = (M * np.log(pi) + (sample_num * (i) - M)
                    * np.log(1 - pi))
        mbic = term1 + term2 + term3 + term4 + term5 + term6 + term7
        return mbic

    def _get_break_points_for_each_k(self, paths, i):
        break_points = []
        break_points.append(paths.shape[1])

        j = paths.shape[1] - 1
        while i >= 0:
            break_points.append(paths[i][j])
            j = paths[i][j] - 1
            i = i - 1
        if break_points[-1] != 0:
            break_points.append(0)
        break_points.reverse()
        return break_points

    def _cal_cnv_for_each_chrom(self, Y, nor_Y):
        nor_Y.columns = Y.columns
        nor_Y.index = Y.index
        bin_num = Y.shape[0]
        # mBIC = np.zeros((bin_num - 1, bin_num))
        # paths = np.zeros((bin_num - 1, bin_num), dtype=np.int64)
        mBIC = np.zeros((30, bin_num))
        paths = np.zeros((30, bin_num), dtype=np.int64)
        loglikeijs = np.zeros((bin_num, bin_num))
        cumsum_Y = np.cumsum(Y).values
        cumsum_nor_Y = np.cumsum(nor_Y).values

        for i in range(bin_num):
            for j in range(i, bin_num):
                if i == 0:
                    row_sum_Y = cumsum_Y[j]
                    row_sum_nor_Y = cumsum_nor_Y[j]
                else:
                    row_sum_Y = cumsum_Y[j] - cumsum_Y[i - 1]
                    row_sum_nor_Y = cumsum_nor_Y[j] - cumsum_nor_Y[i - 1]
                row_sum_Y[row_sum_nor_Y < 20] = 20
                row_sum_nor_Y[row_sum_nor_Y < 20] = 20
                loglikeij = np.sum((1 - np.round(row_sum_Y / row_sum_nor_Y * 2) / 2) * row_sum_nor_Y
                                    + np.log((np.round(row_sum_Y / row_sum_nor_Y * 2) + 1e-04) / 2.0001) * row_sum_Y)
                loglikeijs[i][j] = loglikeij
        k1 = 3/2
        k2 = 2.27
        i = 0
        count = 0
        while True:
            # if i >= bin_num - 1:
            if i >= 30 or i >= bin_num - 1:
                break
            for j in range(i + 1, bin_num):
                max_mbic = None

                # cal all possible mbics for current bin size
                for index in np.arange(i, j):
                    if i == 0:
                        term1 = loglikeijs[i][index] + loglikeijs[index + 1][j]
                    else:
                        term1 = mBIC[i-1][index] + loglikeijs[index + 1][j]

                    term2 = -np.log(special.binom(j+1, i + 1))
                    term3 = -(i + 1) * (k1 - k2)
                    mbic = term1 + term2 + term3
                    if index == i or mbic >= max_mbic:
                        mBIC[i][j] = mbic
                        paths[i][j] = index + 1
                        max_mbic = mbic
            last_col = mBIC[:, -1]
            if i != 0 and count == 0:
                if last_col[i] < last_col[i - 1]:
                    count = count + 1
            if count != 0:
                count = count + 1
            if count > 5:
                break
            i = i + 1

        max_k = i
        break_points_of_all_k = []
        mbic_of_all_k = []
        for i in range(max_k):
            break_points = self._get_break_points_for_each_k(paths, i)
            segs = self._get_segments(break_points)
            mbic = self._cal_mbic(Y, nor_Y, segs)
            break_points_of_all_k.append(break_points)
            mbic_of_all_k.append(mbic)
        if self.K:
            max_mbic_index = self.K - 2
        else:
            max_mbic_index = mbic_of_all_k.index(max(mbic_of_all_k))
        break_points = break_points_of_all_k[max_mbic_index]
        # cal cnv for each segment
        index_Y = np.column_stack(np.where(Y <= 20))
        for index in index_Y:
            Y.iat[index[0], index[1]] = 20
        index_nor_Y = np.column_stack(np.where(nor_Y <= 20))
        for index in index_nor_Y:
            nor_Y.iat[index[0], index[1]] = 20
        cumsum_Y = np.cumsum(Y).values
        cumsum_nor_Y = np.cumsum(nor_Y).values
        cnv = np.zeros(Y.shape, dtype=np.int64)
        for i in range(len(break_points) - 1):
            start = break_points[i]
            end = break_points[i+1] - 1
            if start == 0:
                sum_Y = cumsum_Y[end]
                sum_nor_Y = cumsum_nor_Y[end]
            else:
                sum_Y = cumsum_Y[end] - cumsum_Y[start - 1]
                sum_nor_Y = cumsum_nor_Y[end] - cumsum_nor_Y[start - 1]
            rate = np.round((sum_Y / sum_nor_Y) * 2)
            rate[rate > 14] = 14

            for j in np.arange(start, end+1):
                cnv[j] = rate
        cnv = pd.DataFrame(cnv, columns=Y.columns, index=Y.index)
        return break_points, cnv

    def call(self, bam_dir, out_dir):
        """call CNV for each chromosome

        Parameters
        ----------
        bam_dir : directory path which contains all BAM files

        out_dir : the output directory
        Returns
        -------
        self
        """
        Y_path = os.path.join(out_dir, 'temp.Y.csv')
        nor_Y_path = os.path.join(out_dir, 'temp.norY.csv')
        ref_path = os.path.join(out_dir, 'temp.ref.csv')
        gini_path = os.path.join(out_dir, 'temp.gini.csv')
        ploidy_path = os.path.join(out_dir, 'temp.ploidy.csv')
        scope_path = os.path.join(out_dir, 'run-scope.R')
        utils.write_scope(scope_path)
        command = 'Rscript {0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10} {11}'.format(scope_path, bam_dir, Y_path, ref_path, gini_path, ploidy_path, nor_Y_path,self.seq, self.reg, self.ref, self.mapq, self.bin_len)
        code = os.system(command)
        if code != 0:
            sys.exit(1)
        tasklogger.log_start('SCYN')
        Y = pd.read_csv(Y_path, index_col=0)
        nor_Y = pd.read_csv(nor_Y_path, index_col=0)
        ref = pd.read_csv(ref_path, index_col=0)
        gini = pd.read_csv(gini_path, index_col=0)
        ploidy = pd.read_csv(ploidy_path, index_col=0)
        self.meta_info = pd.DataFrame(index=['c_gini', 'c_ploidy'], columns=Y.columns)
        self.meta_info.loc['c_gini'] = gini.T.iloc[0].values
        self.meta_info.loc['c_ploidy'] = ploidy.T.iloc[0].values
        self.meta_info = self.meta_info.T
        self._cal_cnv(ref, Y, nor_Y)
        self.bin_info = ref

        # clean up temp files
        utils.clean_up([Y_path, nor_Y_path, ref_path,
                        gini_path, ploidy_path, scope_path])

        tasklogger.log_complete('SCYN')

        return self

reason: import numpy,from scipy