text:
import numpy as np

from fos.actor.primitives import AABBPrimitive
from pyglet.gl import GLfloat
from pyglet.gl import *

class Actor(object):
    """ Define a visualization object in Fos """

    def __init__(self,
                 affine = None,
                 aabb = None,
                 force_center_data = False,
                 **kwargs):
        """ Create an actor

        Parameters
        ----------

        affine : 4x4 array
            the affine is expected to be normal, i.e. it has only
            rotational and translational components, but no shears
            the affine is applied to the input vertices data to position
            the actor in the world space. If set to none, an affine is
            generated to positions the actor optimally for the camera view

        aabb :  (corner1, corner2)
            the axis-aligned bounding box. axis-aligned means aligned
            with the world coordinate system axes

            corner1 : 3x1 array
                bottom-left-front point of the box when look into z direction
            corner2 : 3x1 array
                top-right-back point of the box

            If set to None, an axis aligned boundingbox is computed
            using the input vertices

        force_center_data : boolean
            if set to true, the mean of vertices location is subtracted from
            all the vertices. this is useful to compute a better bounding
            box and if the data has no meaningful affine

        obb : (center, orientation, halfwidths)

            center : 3x1 array
                the center point of the aabb
            orientation : 3x3 array
                orthogonal unit vectors
            halfwidths : 3x1 array
                box halfwidths along each axis

        """

        # data related information
        self.vertices = None
        self.living = False
        self.show_aabb = True

#        self.connectivity = None
#        self.field = None # scalar, vector, tensor
#        self.colormap = None
#        self.texture = None

        # movement related information. use the
#        self.velocity = None
#        self.acceleration = None

        # event related information
#        self.event_queue = None
        # mouse or keyboard events on the actor
#        self.event_handlers = None
        # related: menu options for the actor

    def setup(self):
        """ Data preparation """
        # display lists, vbo
        # prepare different resolutions
        pass

    def update(self, dt):
        """ Update the actor
        dt from the global timer """
        pass

    def draw_aabb(self):
        """ Draw the actor """

        if self.show_aabb:
            glPushMatrix()
            glPolygonMode(GL_FRONT_AND_BACK, GL_LINE)
            glLineWidth(1.0)
            glColor3f(1.0, 1.0, 0.0)
            glEnableClientState(GL_VERTEX_ARRAY)
            glVertexPointer(3, GL_FLOAT, 0, self.aabb.vertices_ptr)
            glDrawElements(self.aabb.mode,self.aabb.indices_nr,self.aabb.type,self.aabb.indices_ptr)
            glDisableClientState(GL_VERTEX_ARRAY)
            glPolygonMode(GL_FRONT_AND_BACK, GL_FILL)
            glPopMatrix()

    def delete(self):
        """ Removing the geometry """
        pass

    def info(self):
        """ Show information about the actor """
        # debug mode
        print "this actor is at ", self
        print "number of vertices", len(self.vertices)
        print "is the actor living ?", self.living
        if not self.aabb is None:
            print "has boundary box", self.aabb

    def to_raytracer_file(self):
        """ Save the geometry to a file readable by a raytracer """
        pass

    def process_pickray(self, near, far):
        """ Process the pick ray like intersecting with the actor """
        pass

    def process_keys(self,symbol,modifiers):
        pass

    def process_mouse_motion(self,x,y,dx,dy):
        pass

    def make_aabb(self, aabb = None, margin = 30):
        """ Make the axis aligned bounding box.

        Parameters
        ----------
        aabb : 2-tuple of numpy arrays of shape(3,)
            Defining the box by left-bottom-front and the top-right-back
            coordinate points. If None, a bounding box based on the
            vertices is computed.
        margin : float
            A margin to be added to the computed bounding box

        """
        # if no aabb is given, compute one
        if aabb == None:
            # compute an axis aligned bounding box
            # based on the vertices
            coord1 = np.array([self.vertices[:,0].min(),
                                self.vertices[:,1].min(),
                                self.vertices[:,2].min()], dtype = np.float32)
            coord2 = np.array([self.vertices[:,0].max(),
                               self.vertices[:,1].max(),
                               self.vertices[:,2].max()], dtype = np.float32)
            self.aabb = AABBPrimitive(blf = coord1, trb = coord2, margin = margin)
        else:
            assert len(aabb) == 2
            # otherwise set to given aabb
            self.aabb = AABBPrimitive(blf = aabb[0], trb = aabb[1], margin = margin)

    def get_aabb_coords(self):
        """
        Returns AABB coordinates blf and trb in world space (using the affine)
        """
        ab1 = self.aabb.coord[0]
        ab2 = self.aabb.coord[1]
        r1 = np.dot(self.affine, np.array( [ab1[0], ab1[1], ab1[2], 1.0] ) )
        r2 = np.dot(self.affine, np.array( [ab2[0], ab2[1], ab2[2], 1.0] ) )
        return (r1[:3], r2[:3])

    def make_obb(self):
        pass
        # just reuse the aabb points
#        leftbottom, righttop = self.aabb
#
#        center = np.mean( np.vstack( (leftbottom, righttop) ), axis = 0)
#        halfwidths = (leftbottom - righttop) / 2.0
#        # in our default network, this obb is axis-aligned, thus the
#        # obb is the identity
#        orientation = np.eye( 3, 3 )
#
#        self.obb = (center, halfwidths, orientation)

    def bounding_sphere(self):
        """ Compute the bounding sphere """
        pass
        # can use PCA?

    def bouding_ellipsoid(self):
        """ Compute the bounding elipsoid """
        pass
        # can use PCA?

    ## affine logic
    ###

    def set_affine(self, affine):
        # update the affine
        print "update affine", self.affine
        self.affine = affine
        self._update_glaffine()

    def scale(self, scale_factor):
        """ Scales the actor by scale factor.
        Multiplies the diagonal of the affine for
        the first 3 elements """
        self.affine[0,0] *= scale_factor
        self.affine[1,1] *= scale_factor
        self.affine[2,2] *= scale_factor
        self._update_glaffine()

    def translate(self, dx, dy, dz):
        """ Translate the actor.
        Remember the OpenGL has right-handed
        coordinate system """
        self.affine[0,3] += dx
        self.affine[1,3] += dy
        self.affine[2,3] += dz
        self._update_glaffine()

    def set_position(self, x, y, z):
        """ Position the actor.
        Remember the OpenGL has right-handed
        coordinate system """
        self.affine[0,3] += x
        self.affine[1,3] += y
        self.affine[2,3] += z
        self._update_glaffine()

    def _update_glaffine(self):
        self.glaffine = (GLfloat * 16)(*tuple(self.affine.T.ravel()))

    # life processes
    ###

    def start(self, lifespan = 10, tickingtime = 2.0):
        print "the actor is alive"
        self.living = True
        self.internal_timestamp = 0.0
        # call self.update every tickingtime

    def stop(self):
        print "the actor stops living"
        self.living = False

    def cont(self):
        print "continue to live happily"
        self.living = True

max_stars_repo_name: fos/fos-legacy

max_issues_repo_name: fos/fos-legacy

max_forks_repo_name: fos/fos-legacy


text:
import pandas as pd
import numpy as np

df = pd.DataFrame  ({
'nome' : ['André', 'Junior', 'Nascimento'],
'idade': [38, 37, 36],
'cidade': ['RJ','SP','MG']
})

print(df)
print('\n')

vetor = np.array([5, 6, 7, 8])
v = pd.Series(vetor)

print(vetor)
print(v)
print('\n\n')
print(df.nome) # retorna somente com a coluna nome
print('\n\n')
print(df.idade.mean()) # Retorna com a média de idade

ext: py

max_stars_repo_path: Tele_Aula 4/DataFrame.py

max_issues_repo_path: Tele_Aula 4/DataFrame.py

max_forks_repo_path: Tele_Aula 4/DataFrame.py

reason: import numpy


text:
# Copyright 2020 The FastEstimator Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
import unittest

import numpy as np
import tensorflow as tf
import torch

import fastestimator as fe
from fastestimator.test.unittest_util import is_equal

class TestGetLr(unittest.TestCase):
    def test_get_lr_tf(self):
        m = fe.build(fe.architecture.tensorflow.LeNet, optimizer_fn=lambda: tf.optimizers.Adam(1e-4))
        b = fe.backend.get_lr(model=m)
        self.assertTrue(np.allclose(b, 1e-4))

        m = fe.build(fe.architecture.tensorflow.LeNet, optimizer_fn=lambda: tf.optimizers.Adam(5e-2))
        b = fe.backend.get_lr(model=m)
        self.assertTrue(np.allclose(b, 5e-2))

    def test_get_lr_torch(self):
        m = fe.build(fe.architecture.pytorch.LeNet, optimizer_fn=lambda x: torch.optim.Adam(params=x, lr=1e-4))
        b = fe.backend.get_lr(model=m)
        self.assertTrue(np.allclose(b, 1e-4))

        m = fe.build(fe.architecture.pytorch.LeNet, optimizer_fn=lambda x: torch.optim.Adam(params=x, lr=5e-2))
        b = fe.backend.get_lr(model=m)
        self.assertTrue(np.allclose(b, 5e-2))


text:
#!/usr/bin/env python3

# Copyright (c) Facebook, Inc. and its affiliates.
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

import glob
import os
import random

import numpy as np
import pytest

try:
    import torch
    import torch.distributed

    import habitat_sim.utils.datasets_download as data_downloader
    from habitat_baselines.common.baseline_registry import baseline_registry
    from habitat_baselines.config.default import get_config

    baseline_installed = True
except ImportError:
    baseline_installed = False

def setup_function(test_trainers):
    # Download the needed datasets
    data_downloader.main(["--uids", "rearrange_task_assets", "--no-replace"])

@pytest.mark.skipif(
    not baseline_installed, reason="baseline sub-module not installed"
)
@pytest.mark.parametrize(
    "config_path,num_updates,target_reward",
    [
        ("habitat_baselines/config/rearrange/ddppo_reach_state.yaml", 40, 5.0),
    ],
)
def test_trainers(config_path, num_updates, target_reward):
    # Remove the checkpoints from previous tests
    for f in glob.glob("data/test_checkpoints/test_training/*"):
        os.remove(f)
    # Setup the training
    config = get_config(
        config_path,
        [
            "NUM_UPDATES",
            num_updates,
            "TOTAL_NUM_STEPS",
            -1.0,
            "CHECKPOINT_FOLDER",
            "data/test_checkpoints/test_training",
        ],
    )
    random.seed(config.TASK_CONFIG.SEED)
    np.random.seed(config.TASK_CONFIG.SEED)
    torch.manual_seed(config.TASK_CONFIG.SEED)
    torch.cuda.manual_seed(config.TASK_CONFIG.SEED)
    torch.backends.cudnn.deterministic = True
    if config.FORCE_TORCH_SINGLE_THREADED and torch.cuda.is_available():
        torch.set_num_threads(1)

    assert (
        config.TRAINER_NAME == "ddppo"
    ), "This test can only be used with ddppo trainer"

    trainer_init = baseline_registry.get_trainer(config.TRAINER_NAME)
    assert trainer_init is not None, f"{config.TRAINER_NAME} is not supported"
    trainer = trainer_init(config)

    # Train
    trainer.train()

    # Gather the data
    deltas = {
        k: ((v[-1] - v[0]).sum().item() if len(v) > 1 else v[0].sum().item())
        for k, v in trainer.window_episode_stats.items()
    }
    deltas["count"] = max(deltas["count"], 1.0)
    reward = deltas["reward"] / deltas["count"]

    # Make sure the final reward is greater than the target
    assert (
        reward >= target_reward
    ), f"reward for task {config_path} was {reward} but is expected to be at least {target_reward}"


max_stars_repo_name: aalto-speech/FinnishXL

max_issues_repo_name: aalto-speech/FinnishXL

max_forks_repo_name: aalto-speech/FinnishXL


text:
# -*- coding: utf-8 -*-
"""1-Step Advantage Actor-Critic agent for episodic tasks in OpenAI Gym.

- Author: Curt Park
- Contact: curt.park@medipixel.io
"""

import argparse
from typing import Tuple

import gym
import numpy as np
import torch
import wandb

from rl_algorithms.common.abstract.agent import Agent
from rl_algorithms.common.helper_functions import numpy2floattensor
from rl_algorithms.registry import AGENTS, build_learner
from rl_algorithms.utils.config import ConfigDict

@AGENTS.register_module
class A2CAgent(Agent):
    """1-Step Advantage Actor-Critic interacting with environment.

    Attributes:
        env (gym.Env): openAI Gym environment
        args (argparse.Namespace): arguments including hyperparameters and training settings
        hyper_params (ConfigDict): hyper-parameters
        network_cfg (ConfigDict): config of network for training agent
        optim_cfg (ConfigDict): config of optimizer
        state_dim (int): state size of env
        action_dim (int): action size of env
        actor (nn.Module): policy model to select actions
        critic (nn.Module): critic model to evaluate states
        actor_optim (Optimizer): optimizer for actor
        critic_optim (Optimizer): optimizer for critic
        episode_step (int): step number of the current episode
        i_episode (int): current episode number
        transition (list): recent transition information

    """

    def __init__(
        self,
        env: gym.Env,
        env_info: ConfigDict,
        args: argparse.Namespace,
        hyper_params: ConfigDict,
        learner_cfg: ConfigDict,
        log_cfg: ConfigDict,
    ):
        """Initialize."""
        Agent.__init__(self, env, env_info, args, log_cfg)

        self.transition: list = list()
        self.episode_step = 0
        self.i_episode = 0

        self.hyper_params = hyper_params
        self.learner_cfg = learner_cfg
        self.learner_cfg.args = self.args
        self.learner_cfg.env_info = self.env_info
        self.learner_cfg.hyper_params = self.hyper_params
        self.learner_cfg.log_cfg = self.log_cfg

        self.learner = build_learner(self.learner_cfg)

    def select_action(self, state: np.ndarray) -> torch.Tensor:
        """Select an action from the input space."""
        state = numpy2floattensor(state, self.learner.device)

        selected_action, dist = self.learner.actor(state)

        if self.args.test:
            selected_action = dist.mean
        else:
            predicted_value = self.learner.critic(state)
            log_prob = dist.log_prob(selected_action).sum(dim=-1)
            self.transition = []
            self.transition.extend([log_prob, predicted_value])

        return selected_action

    def step(self, action: torch.Tensor) -> Tuple[np.ndarray, np.float64, bool, dict]:
        """Take an action and return the response of the env."""

        action = action.detach().cpu().numpy()
        next_state, reward, done, info = self.env.step(action)

        if not self.args.test:
            done_bool = done
            if self.episode_step == self.args.max_episode_steps:
                done_bool = False
            self.transition.extend([next_state, reward, done_bool])

        return next_state, reward, done, info

    def write_log(self, log_value: tuple):
        i, score, policy_loss, value_loss = log_value
        total_loss = policy_loss + value_loss

        print(
            "[INFO] episode %d\tepisode step: %d\ttotal score: %d\n"
            "total loss: %.4f\tpolicy loss: %.4f\tvalue loss: %.4f\n"
            % (i, self.episode_step, score, total_loss, policy_loss, value_loss)
        )

        if self.args.log:
            wandb.log(
                {
                    "total loss": total_loss,
                    "policy loss": policy_loss,
                    "value loss": value_loss,
                    "score": score,
                }
            )

    def train(self):
        """Train the agent."""
        # logger
        if self.args.log:
            self.set_wandb()
            # wandb.watch([self.actor, self.critic], log="parameters")

        for self.i_episode in range(1, self.args.episode_num + 1):
            state = self.env.reset()
            done = False
            score = 0
            policy_loss_episode = list()
            value_loss_episode = list()
            self.episode_step = 0

            while not done:
                if self.args.render and self.i_episode >= self.args.render_after:
                    self.env.render()

                action = self.select_action(state)
                next_state, reward, done, _ = self.step(action)
                self.episode_step += 1

                policy_loss, value_loss = self.learner.update_model(self.transition)

                policy_loss_episode.append(policy_loss)
                value_loss_episode.append(value_loss)

                state = next_state
                score += reward

            # logging
            policy_loss = np.array(policy_loss_episode).mean()
            value_loss = np.array(value_loss_episode).mean()
            log_value = (self.i_episode, score, policy_loss, value_loss)
            self.write_log(log_value)

            if self.i_episode % self.args.save_period == 0:
                self.learner.save_params(self.i_episode)
                self.interim_test()

        # termination
        self.env.close()
        self.learner.save_params(self.i_episode)
        self.interim_test()

max_stars_repo_name: mshukor/DSACfD


text:
import torch
from torch.utils.data import Dataset, DataLoader
import numpy as np
from fast_jtnn import *
from fast_molvae import *
import pickle as pickle
import os, random
from tqdm import tqdm

class MolTreeWithPropertyFolder(object):
    def __init__(
        self,
        smiles_folder,
        mol_folder,
        scaffold_folder,
        attr_folder,
        vocab,
        batch_size,
        num_workers=4,
        shuffle=False,
        assm=True,
        replicate=None,
        with_attr=False,
    ):
        self.smiles_folder = smiles_folder
        self.mol_folder = mol_folder
        self.scaffold_folder = scaffold_folder
        self.attr_folder = attr_folder
        self.data_files = [fn for fn in os.listdir(mol_folder)]
        self.data_files.sort(key=lambda k: int(k[:-4].split("-")[-1]))

        self.batch_size = batch_size
        self.vocab = vocab
        self.num_workers = num_workers
        self.shuffle = shuffle
        self.assm = assm

        if replicate is not None:  # expand is int
            self.data_files = self.data_files * replicate

    def __iter__(self):
        for fn in self.data_files:
            sf = os.path.join(self.smiles_folder, fn)
            mf = os.path.join(self.mol_folder, fn)
            scf = os.path.join(self.scaffold_folder, fn)
            pf = os.path.join(self.attr_folder, fn)

            with open(sf, "rb") as f:
                smiles_data = pickle.load(f)
            with open(mf, "rb") as f:
                mol_data = pickle.load(f)
            with open(scf, "rb") as f:
                scaffold_data = pickle.load(f)
            with open(pf, "rb") as f:
                attr_data = pickle.load(f)

            if self.shuffle:
                zipped = list(zip(smiles_data, mol_data, scaffold_data, attr_data))
                random.shuffle(zipped)
                smiles_data, mol_data, scaffold_data, attr_data = zip(*zipped)

            smiles_batches = [
                smiles_data[i : i + self.batch_size]
                for i in range(0, len(smiles_data), self.batch_size)
            ]
            if len(smiles_batches[-1]) < self.batch_size:
                smiles_batches.pop()

            mol_batches = [
                mol_data[i : i + self.batch_size]
                for i in range(0, len(mol_data), self.batch_size)
            ]
            if len(mol_batches[-1]) < self.batch_size:
                mol_batches.pop()

            scaffold_batches = [
                scaffold_data[i : i + self.batch_size]
                for i in range(0, len(scaffold_data), self.batch_size)
            ]
            if len(scaffold_batches[-1]) < self.batch_size:
                scaffold_batches.pop()

            attr_batches = [
                attr_data[i : i + self.batch_size]
                for i in range(0, len(attr_data), self.batch_size)
            ]
            if len(attr_batches[-1]) < self.batch_size:
                attr_batches.pop()

            dataset = MolTreeWithPropertyDataset(
                smiles_batches,
                mol_batches,
                scaffold_batches,
                attr_batches,
                self.vocab,
                self.assm,
            )
            dataloader = DataLoader(
                dataset, batch_size=1, shuffle=False, collate_fn=lambda x: x[0]
            )  # , num_workers=self.num_workers)

            for b in dataloader:
                yield b

            del (
                smiles_data,
                mol_data,
                scaffold_data,
                attr_data,
                smiles_batches,
                mol_batches,
                attr_batches,
                dataset,
                dataloader,
            )

class MolTreeWithPropertyDataset(Dataset):
    def __init__(
        self, smiles_data, mol_data, scaffold_data, attr_data, vocab, assm=True
    ):
        self.smiles_data = smiles_data
        self.mol_data = mol_data
        self.scaffold_data = scaffold_data
        self.attr_data = attr_data
        self.vocab = vocab
        self.assm = assm

        assert len(self.mol_data) == len(self.attr_data)

    def __len__(self):
        return len(self.mol_data)

    def __getitem__(self, idx):
        return (
            self.smiles_data[idx],
            None,  # tensorize(self.mol_data[idx], self.vocab, assm=self.assm),
            self.scaffold_data[idx],
            torch.tensor(self.attr_data[idx]).unsqueeze(1),
        )

def tensorize(tree_batch, vocab, assm=True):
    set_batch_nodeID(tree_batch, vocab)
    smiles_batch = [tree.smiles if tree is not None else None for tree in tree_batch]

    jtenc_holder, mess_dict = JTNNEncoder.tensorize(tree_batch)
    # if jtenc_holder is None: return None
    jtenc_holder = jtenc_holder
    mpn_holder = MPN.tensorize(smiles_batch)

    if assm is False:
        return tree_batch, jtenc_holder, mpn_holder

    cands = []
    batch_idx = []
    for i, mol_tree in enumerate(tree_batch):
        for node in mol_tree.nodes:
            # Leaf node's attachment is determined by neighboring node's attachment
            if node.is_leaf or len(node.cands) == 1:
                continue
            cands.extend([(cand, mol_tree.nodes, node) for cand in node.cands])
            batch_idx.extend([i] * len(node.cands))

    if len(cands) == 0:
        return None
    jtmpn_holder = JTMPN.tensorize(cands, mess_dict)
    batch_idx = torch.LongTensor(batch_idx)

    return tree_batch, jtenc_holder, mpn_holder, (jtmpn_holder, batch_idx)

def set_batch_nodeID(mol_batch, vocab):
    tot = 0
    for mol_tree in mol_batch:
        # if mol_tree is None: continue
        for node in mol_tree.nodes:
            node.idx = tot
            """
            if not vocab.exists(node.smiles):
                print("Node not in vocab")
                node.wid = -1
                continue
            """
            node.wid = vocab.get_index(node.smiles)
            tot += 1

class FlowDataset(Dataset):
    def __init__(
        self,
        smiles_path,
        mol_path,
        scaffold_path,
        attr_path,
        vocab,
        model,
        save_path,
        use_logvar=False,
        load=False,
    ):
        sufix = (
            attr_path.split("/")[-2]
            if attr_path.endswith("/")
            else attr_path.split("/")[-1]
        )
        self.use_logvar = use_logvar
        if load:
            device = next(model.parameters()).device
            self.S = torch.load(f"{save_path}/S.pt", map_location=device)
            self.W_tree = torch.load(
                f"{save_path}/W_tree.pt", map_location=device
            ).float()
            self.W_mol = torch.load(
                f"{save_path}/W_mol.pt", map_location=device
            ).float()
            self.Sc = torch.load(f"{save_path}/Sc.pt", map_location=device).float()
            self.A = torch.load(
                f"{save_path}/A_{sufix}.pt", map_location=device
            ).float()

            if use_logvar:
                self.W_tree_logvar = torch.load(
                    f"{save_path}/W_tree_logvar.pt", map_location=device
                ).float()
                self.W_mol_logvar = torch.load(
                    f"{save_path}/W_mol_logvar.pt", map_location=device
                ).float()
        else:
            S, W_tree, W_mol, Sc, A = list(), list(), list(), list(), list()
            if use_logvar:
                W_tree_logvar, W_mol_logvar = list(), list()
            loader = MolTreeWithPropertyFolder(
                smiles_path, mol_path, scaffold_path, attr_path, vocab, batch_size=32
            )
            iterator = iter(loader)
            model.eval()
            with torch.no_grad():
                for batch in tqdm(loader):

                    s_batch, x_batch, sc_batch, a_batch = batch
                    x_batch, x_jtenc_holder, x_mpn_holder, x_jtmpn_holder = x_batch
                    x_tree_vecs, _, x_mol_vecs = model.encode(
                        x_jtenc_holder, x_mpn_holder
                    )
                    S += [s_batch]
                    W_tree += [model.T_mean(x_tree_vecs)]
                    W_mol += [model.G_mean(x_mol_vecs)]
                    Sc += [sc_batch]
                    A += [a_batch]
                    if use_logvar:
                        W_tree_logvar += [model.T_var(x_tree_vecs)]
                        W_mol_logvar += [model.G_var(x_mol_vecs)]
            del loader
            self.S = np.concatenate(S)
            self.W_tree = torch.cat(W_tree)
            self.W_mol = torch.cat(W_mol)
            self.Sc = torch.from_numpy(np.concatenate(Sc)).float().unsqueeze(-1)
            self.A = torch.cat(A)
            torch.save(self.W_tree, f"{save_path}/W_tree.pt")
            torch.save(self.W_mol, f"{save_path}/W_mol.pt")
            torch.save(self.Sc, f"{save_path}/Sc.pt")
            torch.save(self.A, f"{save_path}/A_{sufix}.pt")
            torch.save(self.S, f"{save_path}/S.pt")
            if use_logvar:
                self.W_tree_logvar = torch.cat(W_tree_logvar)
                self.W_mol_logvar = torch.cat(W_mol_logvar)
                torch.save(self.W_tree_logvar, f"{save_path}/W_tree_logvar.pt")
                torch.save(self.W_mol_logvar, f"{save_path}/W_mol_logvar.pt")
        if self.A.shape[-1] == 1:
            self.A[self.A >= 100] = np.nan
            print(len(self.A))
            idx = (1 - np.isnan(self.A.cpu())[:, 0]).bool()
            self.S, self.W_tree, self.W_mol, self.Sc, self.A = (
                self.S[idx],
                self.W_tree[idx],
                self.W_mol[idx],
                self.Sc[idx],
                self.A[idx],
            )
            print(len(self.A))
            if use_logvar:
                (self.W_tree_logvar, self.W_mol_logvar,) = (
                    self.W_tree_logvar[idx],
                    self.W_mol_logvar[idx],
                )

    def __len__(self):
        return len(self.S)

    def __getitem__(self, idx):
        if self.use_logvar:
            return (
                self.S[idx],
                (self.W_tree[idx], self.W_tree_logvar[idx]),
                (self.W_mol[idx], self.W_mol_logvar[idx]),
                self.Sc[idx],
                self.A[idx],
            )
        else:
            return (
                self.S[idx],
                self.W_tree[idx],
                self.W_mol[idx],
                self.Sc[idx],
                self.A[idx],
            )


text:
#!/usr/bin/python

import tecplot
import os
import numpy as np
import math
import string
import glob
from graphicsGenerator import graphicsGenerator
from tecplot.exception import *
from tecplot.constant import *

def postProcessingEuler():
    # Python directory to bin directory
    os.chdir("../bin");

    # Merge Files
    myFlowFiles_ = glob.glob("FlowData_Block*.dat");
    mySurfaceFiles_ = glob.glob("SurfaceFlowData_Block*.dat");

    if len(myFlowFiles_) != 0:
        print("Merging Flow Files..............................................")
        # Create new file
        myNewFileName_ = "FlowFile_AllBlocks";
        myNewFile_ = open(myNewFileName_ + ".dat", "w");

        for i in range(0,len(myFlowFiles_)):
            # Read Flow Files
            myFile_ = open(myFlowFiles_[i],"r");
            myLines_ = myFile_.readlines();
            myFile_.close();

            # Header
            myHeader_ = [];
            for j in range(0,6):
                myHeader_.append(myLines_[j].rstrip());

            # Data
            myDataset_ = tecplot.data.load_tecplot(myFlowFiles_[i], read_data_option=2);
            myX_ = myDataset_.variable(0).values(0).as_numpy_array();
            myY_ = myDataset_.variable(1).values(0).as_numpy_array();
            myZ_ = myDataset_.variable(2).values(0).as_numpy_array();
            myRo_ = myDataset_.variable(3).values(0).as_numpy_array();
            myUu_ = myDataset_.variable(4).values(0).as_numpy_array();
            myVv_ = myDataset_.variable(5).values(0).as_numpy_array();
            myWw_ = myDataset_.variable(6).values(0).as_numpy_array();
            myPp_ = myDataset_.variable(7).values(0).as_numpy_array();
            myCp_ = myDataset_.variable(8).values(0).as_numpy_array();
            myMach_ = myDataset_.variable(9).values(0).as_numpy_array();

            # Number of Elements
            string, string1, string2 = myHeader_[3].split(", ");
            string3, string4 = string1.split("=");
            myElements_ = int(string4);

            # Connectivity
            myConnectivity_ = [];
            for j in range(len(myLines_)-myElements_,len(myLines_)):
                myConnectivity_.append(myLines_[j].rstrip());

            # Write in new file
            if i == 0:
                myNewFile_.write(myHeader_[0] + "\n");
                myNewFile_.write(myHeader_[1] + "\n");
                myNewFile_.write(myHeader_[2] + "\n");
                myNewFile_.write(myHeader_[3] + "\n");
                myNewFile_.write(myHeader_[4] + "\n");
                myNewFile_.write(myHeader_[5] + "\n");
            else:
                myNewFile_.write(myHeader_[2] + "\n");
                myNewFile_.write(myHeader_[3] + "\n");
                myNewFile_.write(myHeader_[4] + "\n");
                myNewFile_.write(myHeader_[5] + "\n");

            for j in range(0,len(myX_)):
                myNewFile_.write(str(myX_[j]) + "\n");

            for j in range(0,len(myY_)):
                myNewFile_.write(str(myY_[j]) + "\n");

            for j in range(0,len(myZ_)):
                myNewFile_.write(str(myZ_[j]) + "\n");

            for j in range(0,len(myRo_)):
                myNewFile_.write(str(myRo_[j]) + "\n");

            for j in range(0,len(myUu_)):
                myNewFile_.write(str(myUu_[j]) + "\n");

            for j in range(0,len(myVv_)):
                myNewFile_.write(str(myVv_[j]) + "\n");

            for j in range(0,len(myWw_)):
                myNewFile_.write(str(myWw_[j]) + "\n");

            for j in range(0,len(myPp_)):
                myNewFile_.write(str(myPp_[j]) + "\n");

            for j in range(0,len(myCp_)):
                myNewFile_.write(str(myCp_[j]) + "\n");

            for j in range(0,len(myMach_)):
                myNewFile_.write(str(myMach_[j]) + "\n");

            for j in range(0,len(myConnectivity_)):
                myNewFile_.write(myConnectivity_[j] + "\n")

            if i == len(myFlowFiles_)-1:
                myNewFile_.close();

                # Load all data from all blocks
                myDataset_ = tecplot.data.load_tecplot(myNewFileName_ + ".dat", read_data_option=2);

                # Save in szl format
                tecplot.data.save_tecplot_szl(myNewFileName_ + ".szplt", dataset=myDataset_);

                # Delete .dat file
                print("Deleting FlowData_Block*.dat files..............................")
                os.remove( myNewFileName_ + ".dat");

                # Delete FlowDataBlock*.dat for each block
                for i in range(0,len(myFlowFiles_)):
                    os.remove(myFlowFiles_[i]);

                print("Deleting FlowData_Block*.dat files..........................DONE")
                print("Merging Flow Files..........................................DONE")

    if len(mySurfaceFiles_) != 0:
        print("Merging Surface Flow Files......................................")
        # Create new file
        myNewSurfaceFileName_ = "SurfaceFlowFile_AllBlocks";
        myNewFile_ = open(myNewSurfaceFileName_ + ".dat", "w");

        for i in range(0,len(mySurfaceFiles_)):
            # Read Surface Files
            myFile_ = open(mySurfaceFiles_[i],"r");
            myLines_ = myFile_.readlines();
            myFile_.close();

            # Header
            myHeader_ = [];
            for j in range(0,6):
                myHeader_.append(myLines_[j].rstrip());

            # Data
            myDataset_ = tecplot.data.load_tecplot(mySurfaceFiles_[i], read_data_option=2);
            myX_ = myDataset_.variable(0).values(0).as_numpy_array();
            myY_ = myDataset_.variable(1).values(0).as_numpy_array();
            myZ_ = myDataset_.variable(2).values(0).as_numpy_array();
            myRo_ = myDataset_.variable(3).values(0).as_numpy_array();
            myUu_ = myDataset_.variable(4).values(0).as_numpy_array();
            myVv_ = myDataset_.variable(5).values(0).as_numpy_array();
            myWw_ = myDataset_.variable(6).values(0).as_numpy_array();
            myPp_ = myDataset_.variable(7).values(0).as_numpy_array();
            myCp_ = myDataset_.variable(8).values(0).as_numpy_array();
            myMach_ = myDataset_.variable(9).values(0).as_numpy_array();

            # Number of Elements
            string, string1, string2 = myHeader_[3].split(", ");
            string3, string4 = string1.split("=");
            myElements_ = int(string4);

            # Connectivity
            myConnectivity_ = [];
            for j in range(len(myLines_)-myElements_,len(myLines_)):
                myConnectivity_.append(myLines_[j].rstrip());

            # Write in new file
            if i == 0:
                myNewFile_.write(myHeader_[0] + "\n");
                myNewFile_.write(myHeader_[1] + "\n");
                myNewFile_.write(myHeader_[2] + "\n");
                myNewFile_.write(myHeader_[3] + "\n");
                myNewFile_.write(myHeader_[4] + "\n");
                myNewFile_.write(myHeader_[5] + "\n");
            else:
                myNewFile_.write(myHeader_[2] + "\n");
                myNewFile_.write(myHeader_[3] + "\n");
                myNewFile_.write(myHeader_[4] + "\n");
                myNewFile_.write(myHeader_[5] + "\n");

            for j in range(0,len(myX_)):
                myNewFile_.write(str(myX_[j]) + "\n");

            for j in range(0,len(myY_)):
                myNewFile_.write(str(myY_[j]) + "\n");

            for j in range(0,len(myZ_)):
                myNewFile_.write(str(myZ_[j]) + "\n");

            for j in range(0,len(myRo_)):
                myNewFile_.write(str(myRo_[j]) + "\n");

            for j in range(0,len(myUu_)):
                myNewFile_.write(str(myUu_[j]) + "\n");

            for j in range(0,len(myVv_)):
                myNewFile_.write(str(myVv_[j]) + "\n");

            for j in range(0,len(myWw_)):
                myNewFile_.write(str(myWw_[j]) + "\n");

            for j in range(0,len(myPp_)):
                myNewFile_.write(str(myPp_[j]) + "\n");

            for j in range(0,len(myCp_)):
                myNewFile_.write(str(myCp_[j]) + "\n");

            for j in range(0,len(myMach_)):
                myNewFile_.write(str(myMach_[j]) + "\n");

            for j in range(0,len(myConnectivity_)):
                myNewFile_.write(myConnectivity_[j] + "\n")

            if i == len(mySurfaceFiles_)-1:
                myNewFile_.close();

                # Load all data from all blocks
                myDataset_ = tecplot.data.load_tecplot(myNewSurfaceFileName_ + ".dat", read_data_option=2);

                # Save in szl format
                tecplot.data.save_tecplot_szl(myNewSurfaceFileName_ + ".szplt", dataset=myDataset_);

                # Delete .dat file
                print("Deleting SurfaceFlowData_Block*.dat files.......................")
                os.remove(myNewSurfaceFileName_ + ".dat");

                # Delete SurfaceFlowDataBlock*.dat for each block
                for i in range(0,len(mySurfaceFiles_)):
                    os.remove(mySurfaceFiles_[i]);

                print("Deleting SurfaceFlowData_Block*.dat files...................DONE")
                print("Merging Surface Flow Files..................................DONE")

    if len(myFlowFiles_) >= 1 and len(mySurfaceFiles_) >= 1:
        # Euler Files
        myInterfaceFile_ = "output_interface";
        myFlowFile_ = myNewFileName_ + ".szplt";
        mySurfaceFlowFile_ = myNewSurfaceFileName_ + ".szplt";
        myConvergenceFile_ = "Convergence.dat";
        myAerodynamicFile_ = "AerodynamicCoefficients.dat";

    else:
        #SU2 Files
        myInterfaceFile_ = "Interface_ONERAM6.txt";
        myFlowFile_ = "flow_3.06.dat";
        mySurfaceFlowFile_ = "surface_flow_3.06.dat";
        myConvergenceFile_ = "history_3.06.dat";
        myAerodynamicFile_ = "aerodynamic_file.dat";

    #Instantiate graphicsGenerator class
    graphicsgenerator_ = graphicsGenerator(myInterfaceFile_, myFlowFile_, mySurfaceFlowFile_, myConvergenceFile_, myAerodynamicFile_);

    #Generate graphs by user comnmands
    graphicsgenerator_.computeGraphics();

if __name__ == '__main__':
    postProcessingEuler()


text:
import numpy as np
import matplotlib.pyplot as plt
import sklearn.preprocessing
from cvxpy import quad_form, multiply, Variable, Problem, Minimize, pos, norm
import random

##############################################################
# Trees
##############################################################

class _TreeNode():

    def __init__(self, left, right, parent, cutoff_id, cutoff_val, prediction):
        self.left = left
        self.right = right
        self.parent = parent
        self.cutoff_id = cutoff_id
        self.cutoff_val = cutoff_val
        self.prediction = prediction

    def _sqsplit(self, xTr,yTr,weights=[]):

        N,D = xTr.shape
        assert D > 0 # must have at least one dimension
        assert N > 1 # must have at least two samples
        if weights == []: # if no weights are passed on, assign uniform weights
            weights = np.ones(N)
        weights = weights/sum(weights) # Weights need to sum to one (we just normalize them)
        bestloss = np.inf
        feature = np.inf
        cut = np.inf

        xTr = xTr.astype(np.longdouble)
        yTr = yTr.astype(np.longdouble)
        weights = weights.astype(np.longdouble)
        for i in range(D):
            order = np.argsort(xTr[:, i])
            ordered_x = xTr.copy()[order]
            ordered_y = yTr.copy()[order]
            ordered_weights = weights[order]
            for j in range(1, N):
                left_x = ordered_x[:j]; right_x = ordered_x[j:]
                left_y = ordered_y[:j]; right_y = ordered_y[j:]
                left_w = ordered_weights[:j]; right_w = ordered_weights[j:]

                current_cut = (left_x[-1, i] + right_x[0, i])/2
                if current_cut == left_x[-1, i]:
                    continue
                else:
                    P_l = np.sum(left_w * left_y) ; P_r = np.sum(right_w * right_y)
                    W_l = np.sum(left_w) ; W_r = np.sum(right_w)
                    Q_l = np.sum(left_w * (left_y)**2) ; Q_r = np.sum(right_w * (right_y)**2)

                    loss = Q_l - (P_l**2 / W_l) + Q_r - (P_r**2 / W_r)
                    if loss < bestloss:
                        cut = current_cut
                        feature = i
                        bestloss = loss

        return feature, cut, bestloss

    def _generate_nodes(self, xTr, yTr, current_depth, maxdepth, weights):
        n = len(yTr)
        labels = list(set(yTr))

        if n == 0:
            self.prediction = None
        elif (yTr == labels[0]).sum() == n or (yTr == labels[1]).sum() == n:
            W = np.sum(weights)
            self.prediction = sum(weights * yTr) / W
        elif current_depth >= maxdepth:
            W = np.sum(weights)
            self.prediction = sum(weights * yTr) / W
        else:
            f,c,b = self._sqsplit(xTr, yTr, weights)
            left_ind = tuple([xTr[:, f] < c]); right_ind = tuple([xTr[:, f] >= c])
            left_y = yTr[left_ind]; right_y = yTr[right_ind]
            left_x = xTr[left_ind]; right_x = xTr[right_ind]
            left_w = weights[left_ind]; right_w = weights[right_ind]
            self.cutoff_id = f ; self.cutoff_val = c

            self.left = _TreeNode(None, None, self, None, None, None)
            self.left._generate_nodes(left_x, left_y, current_depth + 1, maxdepth, left_w)

            self.right = _TreeNode(None, None, self, None, None, None)
            self.right._generate_nodes(right_x, right_y, current_depth + 1, maxdepth, right_w)

    def predict(self, x):
        if self.prediction == None:
            if x[self.cutoff_id] < self.cutoff_val:
                return self.left.predict(x)
            elif x[self.cutoff_id] >= self.cutoff_val:
                return self.right.predict(x)
        else:
            return self.prediction

class CART():
    def __init__(self, maxdepth=np.inf):
        """
        Discription: Initialise a classification and regression tree (CART) model

        Input:
            maxdepth: default to inf, determines the depth of the tree
        """
        self.maxdepth = maxdepth
        self.root = _TreeNode(None, None, None, None, None, None)

    def fit(self, X, y, weights = None):
        """
        Discription: Initialise a classification and regression tree (CART) model

        Input:
            X: (n, d) matrix of data points
            y: (n,) vector of labels
            weights: default to None, used for Adaboost
        """
        n = X.shape[0]
        w = np.ones(n) / float(n) if weights is None else weights
        self.root._generate_nodes(X, y, 1, self.maxdepth, w)

    def predict(self, X):
        """
        Discription: predct y_hat from xTe using decision tree root.

        Input:
            X:  n x d matrix of data points

        Output:
            predictions: n-dimensional vector of predictions
        """

        assert self.root is not None, "fit() the tree before predicting!"

        n = X.shape[0]
        predictions = np.zeros(n)
        for i in range(n):
            x_i = X[i]
            predictions[i] += self.root.predict(x_i)

        return predictions

    def score(self, X, y, metric = 'mse'):
        """
        Description: reports the score of the model for input X and label y

        Input:
            X: the predictors in shape (n, p) where n is the total number of samples
            y: the truth labels
            metric: default to "mse", determines which of {mse or mae} to report

        Output:
            {mse mae}: mse or mae for the input
        """
        assert metric in ['mse', 'mae'], '"score" parameter must be "mse" or "mae".'
        return ((y - self.predict(X))**2).sum() if metric == 'mse' else abs(y - self.predict(X)).sum()

class RandomForest():
    def __init__(self, maxiter = 100, maxdepth=np.inf):
        """
        Description: Initialise a random forest model

        Input:
            maxiter: default to 100, determines the maximum number of trees
            maxdepth: default to inf, determines the depth of each tree
        """
        self.maxiter = maxiter
        self.maxdepth = maxdepth
        self.alphas = None
        self.trees = [] # a list to store all the trees, each entry is the root of the tree

    def fit(self, X, y):
        """
        Description: Fit the random forest with input X and y

        Input:
            X: (n, d) matrix of data points
            y: (n,) vector of labels

        """
        n = X.shape[0]
        for i in range(self.maxiter):
            samples = np.random.choice(n, n, True) # sample from training set, with replacment
            sample_X = X[samples] # get a new set of training X with the sampled indices
            sample_y = y[samples] # get a new set of training y with the sampled indices
            tree = CART(self.maxdepth)
            tree.fit(sample_X, sample_y)
            self.trees.append(tree.root)

    def predict(self, X):
        """
        Description: Evaluates X using trees.

        Input:
            X:      n x d matrix of data points

        Output:
            pred: (n,) dimensional vector of predictions
        """
        m = len(self.trees)
        n,d = X.shape
        pred = np.zeros(n)
        if not isinstance(self.alphas, list):
            self.alphas = np.ones(m) / m

        for i in range(n):
            pred_x = np.zeros(m)
            x_i = X[i]
            for j in range(m):
                pred_x[j] += self.trees[j].predict(x_i)
            pred[i] = np.sum(pred_x * self.alphas)

        return pred

    def score(self, X, y, metric = 'mse'):
        """
        Description: reports the score of the model for input X and label y

        Input:
            X: the predictors in shape (n, p) where n is the total number of samples
            y: the truth labels
            metric: default to "mse", determines which of {mse or mae} to report

        Output:
            {mse mae}: mse or mae for the input
        """
        assert metric in ['mse', 'mae'], '"score" parameter must be "mse" or "mae".'
        return ((y - self.predict(X))**2).sum() if metric == 'mse' else abs(y - self.predict(X)).sum()

class AdaboostTree(RandomForest):

    def __init__(self, maxiter = 100, maxdepth=2):
        """
        Description: Learns a boosted decision tree.
        Input:
            maxiter:  default to 100, determines the maximum number of trees
            maxdepth: default to 2, determines maximum depth of a tree
        """
        super().__init__(maxdepth)
        self.maxdepth = maxdepth
        self.maxiter = maxiter
        self.alphas = []

    def fit(self, X, y):
        assert np.allclose(np.unique(y), np.array([-1,1])), "The labels must be -1 and 1"
        n = X.shape[0]
        self.weights = np.ones(n) / n

        for i in range(self.maxiter):
            tree = CART(self.maxdepth)
            tree.fit(X, y, weights = self.weights)
            preds = np.sign(tree.predict(X))
            epsilon = np.sum(self.weights[preds != y])

            if epsilon < 0.5:
                alpha = .5 * np.log((1 - epsilon)/epsilon)

                self.alphas.append(alpha)
                self.trees.append(tree.root)

                self.weights = (self.weights * np.exp( - alpha * preds * y)) / (2 * np.sqrt((1 - epsilon) * epsilon))
            else:
                break

##############################################################
# Regression
##############################################################
class LinearRegression():

    def __init__(self, method, intercept = True, lmbda = 0):
        """
        Description: Initialise linear regression model

        Input:
            method: {'gradient_descent', 'linear_algebra'} determines the method used to solve for "w"
            intercept: bool, default to True, determines whether a biased is used
            lmbda: default to 0, used for ridge regression
        """
        assert method in ['gradient_descent', 'linear_algebra'], '"method" parameter must be "gradient_descent" or "linear_algebra".'
        self.method = method
        self.intercept = intercept
        self.lmbda = lmbda

    def _add_intercept(self, X):
        interceptor = np.ones((X.shape[0], 1))
        _X = np.append(interceptor, X, axis = 1)
        return _X

    def fit(self, X, y, epochs = None, lr = None):
        """
        Description: train the model with either closed form solution or gradient descent

        Input:
            X: the predictors in shape (n, p) where n is the total number of samples and p is the number of predictors
            y: the truth values of each sample in shape (n,)
            epochs: default to None, determines the number of epochs trained for gradient descent
            lr: default to None, determines the learning rate for gradient descent
        """
        _X = self._add_intercept(X) if self.intercept else X

        if self.method == 'gradient_descent':
            assert isinstance(epochs, int) and isinstance(lr, float), "Specify learning rate and/or epochs"
            self.w = np.random.rand(_X.shape[1])
            m = X.shape[0]
            for i in range(epochs):
                lmb = self.lmbda * np.ones_like(self.w)
                lmb[0] = 0
                gradient = (1/m) * ((_X.T @ (self.predict(X)-y)) + lmb * self.w)
                self.w -= lr * gradient

        elif self.method == 'linear_algebra':
            n, p = _X.shape
            A = np.identity(p)
            A[0,0] = 0 if self.intercept else 1
            self.w = np.linalg.inv((_X.T @ _X) + (self.lmbda * A)) @ (_X.T @ y)

        self.coef_ = self.w[1:] if self.intercept else self.w
        self.intercept_ = self.w[0] if self.intercept else None

    def predict(self, X):
        """
        Description: Predicts the class for each sample in the input

        Input:
            X:  (n, p) normalised matrix

        Output:
            pred: (n,) dimensional vector where each entry is the models predicted class for the corresponding sample from X
        """
        _X = self._add_intercept(X) if self.intercept else X
        pred = _X @ self.w
        return pred

    def score(self, X, y, metric = 'mse'):
        """
        Description: reports the score of the model on input X and y

        Input:
            X: the predictors in shape (n, p) where n is the total number of samples
            y: the truth labels
            metric: default to "mse", determines which of {mse or mae} to report

        Output:
            {mse mae}: mse or mae for the input
        """
        assert metric in ['mse', 'mae'], '"score" parameter must be "mse" or "mae".'
        return ((y - self.predict(X))**2).sum() if metric == 'mse' else abs(y - self.predict(X)).sum()

class RidgeRegression(LinearRegression):

    def __init__(self, method, lmbda, intercept = True):
        """
        Description: Initialise the ridge regression model

        Input:
            method: {'gradient_descent', 'linear_algebra'} determines the method used to solve for "w"
            intercept: bool, default to True, determines whether a biased is used
            lmbda: regularisation parameter, set to 0 == OLS
        """
        super().__init__(method, intercept, lmbda)

##############################################################
# Logistic Regression
##############################################################

class LogisticRegression():

    def __init__(self, classes, intercept = True):
        """
        Description: Initialise logistic regression model

        Input:
            classes:  number of classes in the dataset
            intercept: default True, add intercept to model
        """
        self.classes = classes
        self.intercept = intercept

    def _add_intercept(self, X):
        interceptor = np.ones((X.shape[0], 1))
        _X = np.append(interceptor, X, axis = 1)
        return _X

    def _softmax(self, x):
        return np.exp(x) / sum(np.exp(x))

    def _grad(self, X, y, batch_size):
        gradient = np.zeros((self.w.shape))
        for i in range(batch_size):
            yi = y[[i], :].T ; xi = X[[i], :]
            yi_hat = self._softmax( self.w @ xi.T )
            gradient += (yi_hat - yi) @ xi
        return gradient

    def fit(self, X, y, epochs = 20, lr = 0.1, batch_size = 10):
        """
        Description: Trains the logistic regression model with gradient descent

        Input:
            X:  (n, p) normalised matrix
            y: (n,) dimensional vector where each value should be an integer indicating the class of the sample
            epochs: default to 20, determines the number of epochs to train for
            lr: default to 0.1, determines the learning rate of gradient descent
            batch_size: default to 10, determines the size of each batch for mini batch gradient descent
        """
        ohe = sklearn.preprocessing.OneHotEncoder().fit(y.reshape(-1,1))
        _y = ohe.transform(y.reshape(-1, 1))
        _X = self._add_intercept(X) if self.intercept else X
        n, p = _X.shape
        self.w = np.zeros((self.classes, p))

        assert batch_size <= n, 'Batch size must be smaller than the number of samples'
        for epoch in range(epochs):
            order = list(np.random.permutation(n))
            for i in range(n // batch_size):
                batch = order[i * batch_size: (i + 1) * batch_size]
                batched_x = _X[batch]
                batched_y = _y[batch]
                batch_gradient = self._grad(batched_x, batched_y,batch_size)
                self.w -= lr * batch_gradient / batch_size

        (self.coef_, self.intercept_) =  (self.w[1:], self.w[0]) if self.intercept else (self.w[1:], None)

    def predict(self, X):
        """
        Description: Predicts the class for each sample in the input

        Input:
            X:  (n, p) normalised matrix

        Output:
            pred: (n,) dimensional vector where each entry is the models predicted class for the corresponding sample from X
        """
        _X = self._add_intercept(X) if self.intercept else X
        n, p = _X.shape
        correct = 0
        pred = np.zeros((n, 1))
        for i in range(n):
            xi = _X[[i], :]
            pred[i, 0] = np.argmax(self._softmax( self.w @ xi.T ))

        return pred.flatten().astype(int)

    def score(self, X, y, verbose = False):
        """
        Description: Reports the error rate for the input data and labels

        Input:
            X:  (n, p) normalised matrix
            y: (n,) dimensional vector where each value should be an integer indicating the class of the sample

        Output:
            error_rate: a scalar value indicating the percentage of incorrect predictions
        """
        n = X.shape[0]
        y_pred = self.predict(X)
        incorrect = ((y_pred != y)>0).sum()
        error_rate = incorrect / n
        print(((y_pred - y)>0))
        if verbose:
            print(f"Error Rate: {error_rate * 100:.2f} %",)
        return error_rate

##############################################################
# primal/kernelSVM
##############################################################
class _kernels():
    def __init__(self, *k_param):
        self.kernels = {"rbf": self._rbf}
                        # "polynomial": self._polynomial,
                        # "linear": self._linear}
        self.k_param = k_param

    def _rbf(self, X, Z):
        kpar = self.k_param[0]
        a, b = np.sum(X**2, axis = -1, keepdims = True), np.sum(Z**2, axis = -1, keepdims = True).T
        norm = a + b - 2*(X@Z.T)
        K = np.exp(-kpar * norm)
        return K

    # def _polynomial(self, X, Z):
    #     kpar = self.k_param[0]
    #     K = (1 + X.dot(Z.T))**kpar
    #     return K

    # def _linear(self, X, Z):
    #     K = X.dot(Z.T)
    #     return K

class kernelSVM(_kernels):

    def __init__(self, C, ktype, *k_param):
        super().__init__(*k_param)
        self.ktype = ktype
        self.computeK = self.kernels[self.ktype]
        self.C = C

    def _dualqp(self, K, yTr, C):
        """
        function alpha = dualqp(K,yTr,C)
        constructs the SVM dual formulation and uses a built-in
        convex solver to find the optimal solution.

        Input:
            K     | the (nxn) kernel matrix
            yTr   | training labels (nx1)
            C     | the SVM regularization parameter

        Output:
            alpha | the calculated solution vector (nx1)
        """
        y = yTr.flatten()
        N, _ = K.shape
        alpha = Variable(N)
        outerY = np.outer(y, y)
        G = outerY * K
        objective = 1/2 * quad_form(alpha, G) - sum(alpha)
        constraint = [alpha >= 0, alpha <= C, sum(multiply(alpha, y)) == 0]
        prob = Problem(Minimize(objective), constraint)
        prob.solve()

        return np.array(alpha.value).flatten()

    def _recoverBias(self, K,yTr,alphas,C):
        """
        function bias=recoverBias(K,yTr,alpha,C);
        Solves for the hyperplane bias term, which is uniquely specified by the
        support vectors with alpha values 0<alpha<C

        INPUT:
        K : nxn kernel matrix
        yTr : nx1 input labels
        alpha  : nx1 vector of alpha values
        C : regularization constant

        Output:
        bias : the scalar hyperplane bias of the kernel SVM specified by alphas
        """
        distances = np.abs(alphas - C/2)
        i = np.where(distances == distances.min())[0][0]
        k = K[:, [i]]
        b = yTr[i] - np.vdot(alphas * yTr, k)

        return b

    def fit(self, xTr,yTr):
        """
        function classifier = dualSVM(xTr,yTr,C,ktype,lmbda);
        Constructs the SVM dual formulation and uses a built-in
        convex solver to find the optimal solution.

        Input:
            xTr   | training data (nxd)
            yTr   | training labels (nx1)
        """
        self.xTr, self.yTr = xTr, yTr
        K = self.computeK(self.xTr, self.xTr)
        eps = 0.00001
        K = (K + K.T) / 2 + eps * np.eye(K.shape[0])

        self.alpha = self._dualqp(K, yTr, self.C)
        self.b = self._recoverBias(K, yTr, self.alpha, self.C)

        self.alpha[self.alpha < 0.000001] = 0

    def predict(self, xTe):
        preds = (self.yTr * self.alpha).reshape((1,-1)).dot(self.computeK(self.xTr, xTe)) + self.b
        return preds

    def score(self, X, y, verbose = False):
        """
        Description: Reports the error rate for the input data and labels

        Input:
            X:  (n, p) normalised matrix
            y: (n,) dimensional vector where each value should be an integer indicating the class of the sample

        Output:
            error_rate: a scalar value indicating the percentage of incorrect predictions
        """
        n = X.shape[0]
        y_pred = self.predict(X)
        incorrect = ((y_pred != y)>0).sum()
        error_rate = incorrect / n
        print(((y_pred - y)>0))
        if verbose:
            print(f"Error Rate: {error_rate * 100:.2f} %",)
        return error_rate

class primalSVM():
    def __init__(self, C=1):
        """
        Description: initialise a primal SVM primal SVM.
        Input:
            C : the SVM regularization parameter
        """
        self.C = C
    def fit(self, Xtr, ytr):
        """
        Input:
            xTr   | training data (nxd)
            yTr   | training labels (n)

        Output:
            fun   | usage: predictions=fun(xTe); predictions.shape = (n,)
            wout  | the weight vector calculated by the solver
            bout  | the bias term calculated by the solver
        """
        self.xTr, self.yTr = Xtr, ytr.flatten()
        N, d = Xtr.shape

        w = Variable(d)
        b = Variable(1)
        objective = self.C * sum(pos(1 - multiply(self.yTr, self.xTr @ w + b))) + norm(w, 2)**2
        constraints = [w >= 0]
        prob = Problem(Minimize(objective), constraints)
        prob.solve()

        self.coef_ = w.value
        self.intercept_ = b.value

    def predict(self, X):
        pred = X.dot(self.coef_) + self.intercept_
        return pred

    def score(self, X, y, verbose = False):
        """
        Description: Reports the error rate for the input data and labels

        Input:
            X:  (n, p) normalised matrix
            y: (n,) dimensional vector where each value should be an integer indicating the class of the sample

        Output:
            error_rate: a scalar value indicating the percentage of incorrect predictions
        """
        n = X.shape[0]
        y_pred = self.predict(X)
        incorrect = ((y_pred != y)>0).sum()
        error_rate = incorrect / n
        print(((y_pred - y)>0))
        if verbose:
            print(f"Error Rate: {error_rate * 100:.2f} %",)
        return error_rate

class kernelRidgeRegression(_kernels):
    def __init__(self):
        pass

##############################################################
# Neural Network
##############################################################

class FeedForward():

    def __init__(self, sizes):
        self.num_layers = len(sizes)
        self.sizes = sizes
        self.biases = [np.random.randn(y, 1) for y in sizes[1:]]
        self.weights = [np.random.randn(y, x) for x, y in zip(sizes[:-1], sizes[1:])]

    def sigmoid(self, z):
        return 1.0/(1.0+np.exp(-z))

    def sigmoid_prime(self, z):
        return self.sigmoid(z)*(1-self.sigmoid(z))

    def feedforward(self, a):
        for b, w in zip(self.biases, self.weights):
            a = self.sigmoid(np.dot(w, a)+b)
        return a

    def fit(self, Xtr, ytr, epochs, mini_batch_size, eta, Xva = None, yva = None):
        training_data = [(x.reshape(-1, 1), y.reshape(-1, 1)) for x, y in zip(Xtr, ytr)]

        n = len(training_data)
        for j in range(epochs):
            random.shuffle(training_data)
            mini_batches = [training_data[k:k+mini_batch_size] for k in range(0, n, mini_batch_size)]
            for mini_batch in mini_batches:
                self.update_mini_batch(mini_batch, eta)
            if Xva is not None and yva is not None:
                validation_data = [(x.reshape(-1, 1), y.reshape(-1, 1)) for x, y in zip(Xva, yva)]
                n_test = len(validation_data)
                print("Epoch {0} validation acc. {1} / {2}".format(j, self.evaluate(validation_data), n_test))
            else:
                print("Epoch {0} complete".format(j))

    def update_mini_batch(self, mini_batch, eta):
        nabla_b = [np.zeros(b.shape) for b in self.biases]
        nabla_w = [np.zeros(w.shape) for w in self.weights]

        for x, y in mini_batch:
            delta_nabla_b, delta_nabla_w = self.backprop(x, y)
            nabla_b = [nb+dnb for nb, dnb in zip(nabla_b, delta_nabla_b)]
            nabla_w = [nw+dnw for nw, dnw in zip(nabla_w, delta_nabla_w)]

        self.weights = [w-(eta/len(mini_batch))*nw for w, nw in zip(self.weights, nabla_w)]
        self.biases = [b-(eta/len(mini_batch))*nb for b, nb in zip(self.biases, nabla_b)]

    def backprop(self, x, y):

        nabla_b = [np.zeros(b.shape) for b in self.biases]
        nabla_w = [np.zeros(w.shape) for w in self.weights]
        activation = x
        activations = [x]
        zs = []
        for b, w in zip(self.biases, self.weights):
            z = np.dot(w, activation)+b
            zs.append(z)
            activation = self.sigmoid(z)
            activations.append(activation)

        delta = self.cost_derivative(activations[-1], y) * self.sigmoid_prime(zs[-1])
        nabla_b[-1] = delta
        nabla_w[-1] = np.dot(delta, activations[-2].transpose())

        for l in range(2, self.num_layers):
            z = zs[-l]
            sp = self.sigmoid_prime(z)
            delta = np.dot(self.weights[-l+1].transpose(), delta) * sp
            nabla_b[-l] = delta
            nabla_w[-l] = np.dot(delta, activations[-l-1].transpose())
        return (nabla_b, nabla_w)

    def evaluate(self, test_data):
        test_results = [(np.argmax(self.feedforward(x)), np.argmax(y)) for (x, y) in test_data]
        return sum(int(x == y) for (x, y) in test_results)

    def cost_derivative(self, output_activations, y):
        return (output_activations-y)

    def predict(self, X):
        data = [x.reshape(-1, 1) for x in X]
        preds = np.array([np.argmax(self.feedforward(x)) for x in data])
        return preds

##############################################################
# k nearest neighbour
##############################################################
class _knn():
    def __init__(self, k, method = 'l2'):
        """
        Description: Initialise the k-nearest neighbour model

        Input:
            k: the number of nearest neighbours to consider when making predictions
        """
        self.method = method
        self.measure_method = {"l2": self._l2distance}
        self.k = k

        assert self.method in list(self.measure_method.keys()), "Use {L2} norm"

    def _l2distance(self, X, Z=None):
        if Z is None:
            Z=X

        n,d1=X.shape
        m,d2=Z.shape
        assert (d1==d2), "Dimensions of input vectors must match!"

        # YOUR CODE HERE
        D = np.zeros((n, m))
        for i in range(m):
            z_i = Z[i]
            D[:, i] = np.linalg.norm(X - z_i, axis = 1)

        return D

    def _findknn(self, Xte):
        D = self.measure_method[self.method](self.Xtr, Xte)
        indices = np.argpartition(D, self.k, axis = 0)[:self.k]
        dists = np.partition(D, self.k, axis = 0)[:self.k]
        return indices, dists

class knnRegression(_knn):
    def __init__(self, k, method = 'l2'):
        """
        Description: Initialise the k-nearest neighbour model

        Input:
            k: the number of nearest neighbours to consider when making predictions
            measure: {L2, L1}default to l2, the metrics used to search for nearest neighbours
        """
        super().__init__(k, method)

    def fit(self, Xtr, ytr):
        self.Xtr = Xtr
        self.ytr = ytr

    def predict(self, Xte):
        """
        Description: Initialise the k-nearest neighbour model

        Input:
            X: (n, d) dimensional matrix of datapoints

        Output:
            preds: predicted class of the corresponding data
        """
        indiced, dists = self._findknn(Xte)

        n,d = Xte.shape
        preds = np.zeros(n)
        ind = self._findknn(Xte)[0]
        for i in range(n):
            indices_i = list(ind[:, i])
            nn_y_i = self.ytr[indices_i]
            preds[i] = nn_y_i.mean()
        return preds

    def score(self, X, y, metric = 'mse'):
        """
        Description: reports the score of the model on input X and y

        Input:
            X: the predictors in shape (n, p) where n is the total number of samples
            y: the truth labels
            metric: default to "mse", determines which of {mse or mae} to report

        Output:
            score: mse or mae for the input
        """
        assert metric in ['mse', 'mae'], '"score" parameter must be "mse" or "mae".'
        score = ((y - self.predict(X))**2).sum() if metric == 'mse' else abs(y - self.predict(X)).sum()
        return score

class knnClassifier(_knn):
    def __init__(self, k, method = 'l2'):
        """
        Description: Initialise the k-nearest neighbour model

        Input:
            X: (n, d) dimensional matrix of datapoints
        """
        super().__init__(k, method)

    def _mode(self, array):
        unique = list(set(array))
        mode = None
        count = np.NINF
        for i in unique:
            current_count = (array == i).sum()
            if current_count > count:
                count = current_count
                mode = i

        return mode

    def fit(self, Xtr, ytr):
        self.Xtr = Xtr
        self.ytr = ytr

    def predict(self, Xte):
        """
        Description: Initialise the k-nearest neighbour model

        Input:
            X: (n, d) dimensional matrix of datapoints

        Output:
            preds: predicted class of the corresponding data
        """
        indiced, dists = self._findknn(Xte)

        n,d = Xte.shape
        preds = np.zeros(n)
        ind = self._findknn(Xte)[0]
        for i in range(n):
            indices_i = list(ind[:, i])
            nn_y_i = self.ytr[indices_i]
            preds[i] = self._mode(nn_y_i)
        return preds

    def score(self, X, y, verbose = False):
        """
        Description: Reports the error rate for the input data and labels

        Input:
            X:  (n, p) normalised matrix
            y: (n,) dimensional vector where each value should be an integer indicating the class of the sample

        Output:
            error_rate: a scalar value indicating the percentage of incorrect predictions
        """
        n = X.shape[0]
        y_pred = self.predict(X)
        incorrect = ((y_pred != y)>0).sum()
        error_rate = incorrect / n
        if verbose:
            print(f"Error Rate: {error_rate * 100:.2f} %",)
        return error_rate

reason: import numpy,from cvxpy


text:
# import the necessary packages
from imagesearch.transform import four_point_transform
from skimage.filters import threshold_local
import numpy as np
import argparse
import cv2
import imutils

# construct the argument parser and parse the arguments
ap = argparse.ArgumentParser()
ap.add_argument("-i", "--image", required = True,
	help = "Path to the image to be scanned")
args = vars(ap.parse_args())

# load the image and compute the ratio of the old height
# to the new height, clone it, and resize it
image = cv2.imread(args["image"])
ratio = image.shape[0]/500.0
orig = image.copy()
image = imutils.resize(image,height=500)

# convert image to grayscale , blur, and find edges
gray = cv2.cvtColor(image,cv2.COLOR_BGR2GRAY)
gray = cv2.GaussianBlur(gray, (5, 5), 0)
edged = cv2.Canny(gray,75,200)

# show the original image and the edge detected image
print("STEP 1: Edge Detection")
cv2.imshow("Image",image)
cv2.imshow("Edged", edged)
cv2.waitKey(0)
cv2.destroyAllWindows()

# finding contours in the image , keeping the largest ones
cnts = cv2.findContours(edged.copy(), cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
cnts = imutils.grab_Ccontours(cnts)
cnts  =sorted(cnts,key=cv2.contourAre,reverse=True)[:5]

# loop over the contours
for c in cnts:
	# approximate contour
	peri = cv2.arcLength(c,True)
	approx = cv2.approxPolyDP(c,0.02*peri,True)

	# if out contour has four points then we can assume it to be paper
	if len(approx)==4:
		screenCnt = approx
		break

# show contour (outline) of the paper
print("STEP 2: Find contours of paper")
cv2.drawContours(image, [screenCnt], -1, (0, 255, 0), 2)
cv2.imshow("Outline",image)
cv2.waitKey(0)
cv2.destroyAllWindows()

# apply four point transform to obtain top-down view of image
wraped = four_point_transform(orig,screenCnt.reshape(4,2)*ratio)

# convert wrapped image to grayscale, threshold it and give paper effect
wraped = cv2.cvtColor(wraped,cv2.COLOR_BGR2GRAY)
T = threshold_local(wraped,11,offset = 10,method="gaussian")
wraped = (wraped>T).astype("Unit8") * 255

# show the original and scanned images
print("STEP 3: Apply perspective transform")

cv2.imshow("Original", imutils.resize(orig, height = 650))
cv2.imshow("Scanned", imutils.resize(warped, height = 650))
cv2.waitKey(0)


text:
import random
import numpy as np
from collections import deque

class ReplayBuffer(object):
    def __init__(self, capacity):
        self.buffer = deque(maxlen=capacity)

    def push(self, state, action, reward, next_state, done):
        state = np.expand_dims(state, 0)
        next_state = np.expand_dims(next_state, 0)

        self.buffer.append((state, action, reward, next_state, done))

    def sample(self, batch_size):
        state, action, reward, next_state, done = zip(
            *random.sample(self.buffer, batch_size)
        )
        return np.concatenate(state), action, reward, np.concatenate(next_state), done

    def __len__(self):
        return len(self.buffer)

max_stars_repo_name: ksang/Voigt-Kampff

max_issues_repo_name: ksang/Voigt-Kampff

max_forks_repo_name: ksang/Voigt-Kampff


text:
import numpy as np
from typing import Callable, Iterable, Optional

from emulation._typing import FortranState

Mask = Callable[[FortranState, FortranState], FortranState]

def compose_masks(funcs: Iterable[Mask]) -> Mask:
    """Compose multiple masks first masks are applied first"""

    func_list = list(funcs)

    def composed(state: FortranState, emulator: FortranState):
        out: FortranState = emulator
        for func in func_list:
            out = func(state, out)
        return out

    return composed

class RangeMask:
    def __init__(
        self, key: str, min: Optional[float] = None, max: Optional[float] = None
    ) -> None:
        self.min = min
        self.max = max
        self.key = key

    def __call__(self, state: FortranState, emulator: FortranState) -> FortranState:
        out = {**emulator}
        if self.min is not None:
            out[self.key] = np.maximum(out[self.key], self.min)

        if self.max is not None:
            out[self.key] = np.minimum(out[self.key], self.max)

        return out

class LevelMask:
    def __init__(self, key: str, start: Optional[int], stop: Optional[int]):
        self.key = key
        self.start = start
        self.stop = stop

    def __call__(self, state: FortranState, emulator: FortranState) -> FortranState:
        use_fortran_state = slice(self.start, self.stop)
        # Fortran state TOA is index 79, and dims are [z, sample]
        emulator_field = np.copy(emulator[self.key])
        emulator_field[use_fortran_state] = state[self.key][use_fortran_state]
        return {**emulator, self.key: emulator_field}


text:
import numpy as np
import shutil
import pandas as pd

from sqlalchemy import create_engine
from sqlalchemy_utils import database_exists, create_database
import psycopg2

def productIDs():
    """
    dummy product replacement for real product for privacy purposes.
    """
    return ['prod1','prod2']

def price_product_dict():
    fname = '../../original_data/productprices.csv'
    df = pd.read_csv(fname)
    price_dict = {df['product'][i]:df['prices'][i]
                for i in range(len(df['prices']))}
    return price_dict

def get_features(df):
    X = df[['delta_price','buybox','sessions','pageviews']]
    y = df['demand']
    return df['weeks'],X,y

def update_sales(datafile_path):
    # copy old file and save it for backup
    current_sales_fname = datafile_path + '/sales.csv'
    backup_sale_fname = datafile_path + '/sales_backup.csv'
    shutil.copy(current_sales_fname,backup_sale_fname)
    df_sales = pd.read_csv(current_sales_fname)

    new_sales_fname = datafile_path + '/sales_thisweek.csv'
    df_new = pd.read_csv(new_sales_fname)
    df_new['weeks'] = np.max(df_sales['weeks']) + 1

    # update sales
    df = pd.concat([df_sales,df_new])

    # Save most up to date sales data
    df.to_csv(current_sales_fname)
    return df

def sql_select_data(prod_id):
    dbname,username,pswd = read_credential()

    engine = create_engine('postgresql://%s:%s@localhost/%s'%(username,pswd,dbname))

    # connect to database.
    con = None
    con = psycopg2.connect(database = dbname, user = username, host='localhost', password=pswd)

    # query:
    sql_query = """
    SELECT * FROM sales_data_table WHERE product_id='%s';
    """ % prod_id

    sales_data = pd.read_sql_query(sql_query,con)
    return sales_data

def read_credential():
    fname = './sql_credential.dat'
    credential = []
    with open(fname) as f:
        for line in f:
            line = line.rstrip().split(' ')
            credential.append(line[1])
    return credential


text:
#!/usr/bin/env python
"""
converts a homer motif to a jaspar motif file
"""

### imports ###
import sys
import numpy as np

if __name__ == '__main__':
    if len(sys.argv) < 3:
        print('Usage')
        print('homer2jaspar.py <homer_motif> <output_file_path>')
        sys.exit(0)
    else:
        homer_path= sys.argv[1]
        output_path = sys.argv[2]

    with open(homer_path) as f:
        data = f.readlines()

    name_line = data[0]
    name_tokens = name_line.strip().split('\t')
    motif_id = name_tokens[0][1:]
    motif_name = name_tokens[1]

    A_freqs = []
    C_freqs = []
    G_freqs = []
    T_freqs = []

    for line in data[1:]:
        tokens = line.strip().split('\t')
        A_freqs.append(str(int(float(tokens[0])*10000)))
        C_freqs.append(str(int(float(tokens[1])*10000)))
        G_freqs.append(str(int(float(tokens[2])*10000)))
        T_freqs.append(str(int(float(tokens[3])*10000)))

    out_file = open(output_path, 'w')

    out_file.write('>'+motif_id + '\t' + motif_name + '\n')
    out_file.write('A [ ' + ' '.join(A_freqs) + ' ]\n')
    out_file.write('C [ ' + ' '.join(C_freqs) + ' ]\n')
    out_file.write('G [ ' + ' '.join(G_freqs) + ' ]\n')
    out_file.write('T [ ' + ' '.join(T_freqs) + ' ]\n')
    out_file.close()

max_stars_repo_name: jenhantao/abtba

max_issues_repo_name: jenhantao/abtba

max_forks_repo_name: jenhantao/abtba


text:
#  Copyright 2019-2020 The Lux Authors.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

from .context import lux
import pytest
import pandas as pd
import numpy as np
from tempfile import TemporaryDirectory
from pathlib import Path

def test_head_tail(global_var):
    df = pytest.car_df
    df._ipython_display_()
    assert df._message.to_html() == ""
    df.head()._ipython_display_()
    assert (
        "Lux is visualizing the previous version of the dataframe before you applied head."
        in df._message.to_html()
    )
    df._ipython_display_()
    assert df._message.to_html() == ""
    df.tail()._ipython_display_()
    assert (
        "Lux is visualizing the previous version of the dataframe before you applied tail."
        in df._message.to_html()
    )

def test_describe(global_var):
    df = pytest.college_df
    summary = df.describe()
    summary._ipython_display_()
    assert len(summary.columns) == 10

def test_groupby_describe(global_var):
    df = pytest.college_df
    result = df.groupby("FundingModel")["AdmissionRate"].describe()
    result._ipython_display_()
    assert result.shape == (3, 8)

def test_convert_dtype(global_var):
    df = pytest.college_df
    cdf = df.convert_dtypes()
    cdf._ipython_display_()
    assert list(cdf.recommendation.keys()) == ["Correlation", "Distribution", "Occurrence"]

def test_infs():
    nrows = 100_000

    # continuous
    c1 = np.random.uniform(0, 1, size=nrows)
    c1[2] = np.inf
    c2 = np.random.uniform(0, 1, size=nrows)
    c2[3] = np.inf

    # discrete
    d1 = np.random.randint(0, 2, size=nrows)
    d2 = np.random.randint(0, 2, size=nrows)

    df = pd.DataFrame({"c1": c1, "c2": c2, "d1": d1, "d2": d2})

    df._ipython_display_()

def test_timedeltas():
    nrows = 100_000

    c1 = np.random.uniform(0, 10, size=nrows)
    c2 = c1.astype("timedelta64[ms]")

    df = pd.DataFrame({"c1": c1, "c2": c2})

    df._ipython_display_()

def test_datetime_index():
    nrows = 10

    # create a datetime index, freq in seconds
    dt = pd.date_range("1/1/2019", periods=nrows, freq="1s")

    # continuous
    c1 = np.random.uniform(0, 1, size=nrows)

    data = np.random.uniform(0, 1, size=(nrows, 50))
    df = pd.DataFrame(data, index=dt)

    df._ipython_display_()

def test_interval():
    nrows = 100_000

    c1 = pd.Interval(left=0, right=nrows)
    c2 = np.random.uniform(0, 10, size=nrows)

    df = pd.DataFrame({"c1": c1, "c2": c2})

    df._ipython_display_()

def test_datetime_index_serialize():
    nrows = 100000

    # create a datetime index, freq in seconds
    dt = pd.date_range("1/1/2019", periods=nrows, freq="1s")

    # continuous
    c1 = np.random.uniform(0, 1, size=nrows)

    df = pd.DataFrame({"c1": c1}, index=dt)

    with TemporaryDirectory() as tmpdir:
        tmpdir = Path(tmpdir)
        df.to_csv(tmpdir / "test.csv")
        df = pd.read_csv(tmpdir / "test.csv", names=["date", "c1"], index_col="date", header=0)

    df._ipython_display_()

def test_cut_mixed_types():

    a = [1, "2", 3, "4", 5, "6", 7, "8", 9, "10"] * 1000
    b = np.random.uniform(0, 1, size=len(a))

    df = pd.DataFrame({"a": a, "b": b})

    df._ipython_display_()

max_stars_repo_name: hydrosquall/lux


text:
import os, sys
import numpy as np
import cv2

sys.path.append(os.path.expanduser("~/programming/machine_learning/tuberculosis_project/lira_static/"))

import img_handler
from img_handler import *

def recursive_get_paths(img_dir):
    """
    Arguments:
        img_dir : directory to recursively traverse. Should be a string.

    Returns:
        A list of tuples, where each tuple is of the form
            (path and filename str, filename str), e.g.
            ("/home/darkelement/test.txt", "test.txt")
    """
    paths = []
    for (path, dirs, fnames) in os.walk(img_dir):
        for fname in fnames:
            paths.append((os.path.join(path, fname), fname))
    return paths

def load_images(img_dir):
    """
    Arguments:
        img_dir: string directory where all image files are stored.
            All files here should be images, that are readable by OpenCV.

        We furthermore assume these images are RGB images,
            and that each pixel is in the range of 0-255, so that they are of
            type uint8.

    Returns:
        Iterates through each image and adds them all to a numpy memmap, which is returned.
    """
    print "Getting all Images from Input Directory..."
    path_infos = recursive_get_paths(img_dir)

    #Create memmap
    imgs = np.memmap("../data/images.dat", dtype="uint8", mode="w", shape=(
    with h5py.File(archive_dir, mode="w", chunks=True, compression="gzip") as hf:

        """
        Loop through samples
        """
        for sample_i, sample_path_info in enumerate(sample_path_infos):
            sample_path, sample_fname = sample_path_info

            #sys.stdout.write("\rArchiving Image #%i:%s" % (sample_i, sample_fname))
            #sys.stdout.flush()
            print"\rArchiving Image #%i:%s" % (sample_i, sample_fname)

            """
            Get appropriate version of img
            """
            if rgb:
                img = cv2.imread(sample_path)
            else:
                img = cv2.imread(sample_path, 0)

            """
            Archive img
            """
            hf.create_dataset(str(sample_i), data=img)

    print ""#flush formatting


text:
# encoding=utf8

from math import pow
from unittest import TestCase

import numpy as np

from niapy.util.factory import get_problem

class TestProblemFunctions(TestCase):
    """Testing the problems."""

    def setUp(self):
        """Set up the tests."""
        self.dimension = 5
        self.array = np.asarray([0, 0, 0, 0, 0])
        self.array2 = np.asarray([1, 1, 1, 1, 1])
        self.array3 = np.asarray([420.968746, 420.968746, 420.968746, 420.968746, 420.968746])
        self.array4 = np.asarray([-2.903534, -2.903534])
        self.array5 = np.asarray([-0.5, -0.5, -0.5, -0.5, -0.5])
        self.array6 = np.asarray([-1, -1, -1, -1, -1])
        self.array7 = np.asarray([2, 2, 2, 2, 2])
        self.array8 = np.asarray(
            [7.9170526982459462172, 7.9170526982459462172, 7.9170526982459462172, 7.9170526982459462172,
             7.9170526982459462172])
        self.array9 = np.asarray([-5.12, -5.12, -5.12, -5.12, -5.12])
        self.array10 = np.asarray([1, 2, 3, 4, 5])

    def test_rastrigin(self):
        """Test the rastrigin function."""
        rastrigin = get_problem('rastrigin', dimension=self.dimension)
        self.assertEqual(rastrigin(self.array), 0.0)

    def test_rosenbrock(self):
        """Test the rosenbrock function."""
        fun = get_problem('rosenbrock', self.dimension)
        self.assertEqual(fun(self.array2), 0.0)

    def test_griewank(self):
        """Test the griewank function."""
        fun = get_problem('griewank', self.dimension)
        self.assertEqual(fun(self.array), 0.0)

    def test_sphere(self):
        """Test the sphere function."""
        fun = get_problem('sphere', self.dimension)
        self.assertEqual(fun(self.array), 0.0)

    def test_ackley(self):
        """Test the ackley function."""
        fun = get_problem('ackley', self.dimension)
        self.assertAlmostEqual(fun(self.array), 0.0, places=10)

    def test_schwefel(self):
        """Test the schwefel function."""
        fun = get_problem('schwefel', self.dimension)
        self.assertAlmostEqual(fun(self.array3), 0.0, places=3)

    def test_schwefel221(self):
        """Test the schwefel 221 function."""
        fun = get_problem('schwefel221', self.dimension)
        self.assertEqual(fun(self.array), 0.0)

    def test_schwefel222(self):
        """Test the schwefel 222 function."""
        fun = get_problem('schwefel222', self.dimension)
        self.assertEqual(fun(self.array), 0.0)

    def test_whitley(self):
        """Test the whitley function."""
        fun = get_problem('whitley', self.dimension)
        self.assertEqual(fun(self.array2), 0.0)

    def test_styblinski_tang(self):
        """Test the styblinski tang function."""
        fun = get_problem('styblinski_tang', dimension=2)
        self.assertAlmostEqual(fun(self.array4), -78.332, places=3)

    def test_sum_squares(self):
        """Test the sum squares function."""
        fun = get_problem('sum_squares', self.dimension)
        self.assertEqual(fun(self.array), 0.0)

    def test_stepint(self):
        """Test the stepint function."""
        fun = get_problem('stepint', self.dimension)
        self.assertEqual(fun(self.array9), -5.0)

    def test_step(self):
        """Test the step function."""
        fun = get_problem('step', self.dimension)
        self.assertEqual(fun(self.array), 0.0)

    def test_step2(self):
        """Test the step 2 function."""
        fun = get_problem('step2', self.dimension)
        self.assertEqual(fun(self.array5), 0.0)

    def test_step3(self):
        """Test the step3 function."""
        fun = get_problem('step3', self.dimension)
        self.assertEqual(fun(self.array), 0.0)

    def test_schumer_steiglitz(self):
        """Test the schumer steiglitz function."""
        fun = get_problem('schumer_steiglitz', self.dimension, -100, 100)
        self.assertEqual(fun(self.array), 0.0)

    def test_salomon(self):
        """Test the salomon function."""
        fun = get_problem('salomon', self.dimension, -100.0, 100.0)
        self.assertEqual(fun(self.array), 0.0)

    def test_quintic(self):
        """Test the quintic function."""
        fun = get_problem('quintic', self.dimension, -10.0, 10.0)
        self.assertEqual(fun(self.array6), 0.0)

    def test_quintic2(self):
        """Test the quintic function."""
        fun = get_problem('quintic', self.dimension, -10.0, 10.0)
        self.assertEqual(fun(self.array7), 0.0)

    def test_pinter(self):
        """Test the pinter function."""
        fun = get_problem('pinter', self.dimension, -10.0, 10.0)
        self.assertEqual(fun(self.array), 0.0)

    def test_alpine1(self):
        """Test the alpine 1 function."""
        fun = get_problem('alpine1', self.dimension, -10.0, 10.0)
        self.assertEqual(fun(self.array), 0.0)

    def test_alpine2(self):
        """Test the alpine 2 function."""
        fun = get_problem('alpine2', self.dimension, 0.0, 10.0)
        self.assertEqual(fun(self.array8), pow(2.8081311800070053291, self.dimension))

    def test_chung_reynolds(self):
        """Test the chung reynolds function."""
        fun = get_problem('chung_reynolds', self.dimension, -100, 100)
        self.assertEqual(fun(self.array), 0.0)

    def test_csendes(self):
        """Test the csendes function."""
        fun = get_problem('csendes', self.dimension, -1.0, 1.0)
        self.assertEqual(fun(self.array), 0.0)

    def test_bent_cigar(self):
        """Test the bent cigar function."""
        fun = get_problem('bent_cigar', 2, -100, 100)
        self.assertAlmostEqual(fun(np.zeros(2)), 0.0, delta=1e-4)

    def test_discus(self):
        """Test the discus function."""
        fun = get_problem('discus', self.dimension, -100, 100)
        self.assertAlmostEqual(fun(self.array10), 1000054.0, delta=1e-4)

    def test_elliptic(self):
        """Test the elliptic function."""
        fun = get_problem('elliptic', self.dimension, -100, 100)
        self.assertAlmostEqual(fun(self.array10), 5129555.351959938, delta=2e6)

    def test_expanded_griewank_plus_rosenbrock(self):
        """Test the expanded griewank plus rosenbrock function."""
        fun = get_problem('expanded_griewank_plus_rosenbrock', self.dimension, -100, 100)
        self.assertAlmostEqual(fun(self.array), 2.2997, delta=1e2)

    def test_expanded_schaffer(self):
        """Test the expanded schaffer function."""
        fun = get_problem('expanded_schaffer', self.dimension, -100, 100)
        self.assertAlmostEqual(fun(self.array10), 2.616740208857464, delta=1e-4)

    def test_schaffer_n2(self):
        """Test the schaffer n. 2 function."""
        fun = get_problem('schaffer2', lower=-100, upper=100)
        self.assertAlmostEqual(fun(self.array10[:2]), 0.02467, delta=1e-4)

    def test_schaffer_n4(self):
        """Test the schaffer n. 4 function."""
        fun = get_problem('schaffer4', lower=-100, upper=100)
        self.assertAlmostEqual(fun(self.array10[:2]), 0.97545, delta=1e-4)

    def test_hgbat(self):
        """Test the hgbat function."""
        fun = get_problem('hgbat', self.dimension, -100, 100)
        self.assertAlmostEqual(fun(self.array10), 61.91502622129181, delta=60)

    def test_katsuura(self):
        """Test the katsuura function."""
        fun = get_problem('katsuura', self.dimension, -100, 100)
        self.assertAlmostEqual(fun(self.array10), 3837.4739882594373, delta=4000)

    def test_modified_schwefel(self):
        """Test the modified schwefel function."""
        fun = get_problem('modified_schwefel', self.dimension, -100, 100)
        self.assertAlmostEqual(fun(self.array10), 6.9448853328785844, delta=350)

    def test_weierstrass(self):
        """Test the weierstrass function."""
        fun = get_problem('weierstrass', self.dimension, -100, 100)
        self.assertAlmostEqual(fun(self.array10), 0.0, delta=1e-4)

    def test_happy_cat(self):
        """Test the happy cat function."""
        fun = get_problem('happy_cat', self.dimension, alpha=0.125)
        self.assertAlmostEqual(fun(self.array10), 15.1821333, delta=1e-4)

    def test_qing(self):
        """Test the qing function."""
        fun = get_problem('qing', self.dimension, -500, 500)
        self.assertAlmostEqual(fun(self.array10), 584.0, delta=1e-4)

    def test_ridge(self):
        """Test the ridge function."""
        fun = get_problem('ridge', dimension=self.dimension, lower=-64, upper=64)
        self.assertAlmostEqual(fun(self.array10), 371.0, delta=1e-4)

    def test_michalewicz(self):
        """Test the michalewicz function."""
        fun = get_problem('michalewicz', dimension=2, lower=0, upper=np.pi)
        self.assertAlmostEqual(fun(np.asarray([2.20, 1.57])), -1.8013, delta=1e-3)

    def test_levy(self):
        """Test the levy function."""
        fun = get_problem('levy', dimension=2, lower=0, upper=np.pi)
        self.assertAlmostEqual(fun(np.ones(2)), 0.0)

    def test_sphere2(self):
        """Test the sphere 2 function."""
        fun = get_problem('sphere2', dimension=2, lower=-1, upper=1)
        self.assertAlmostEqual(fun(np.zeros(2)), 0.0)

    def test_sphere3(self):
        """Test the sphere 3 function."""
        fun = get_problem('sphere3', dimension=2, lower=-65.536, upper=65.536)
        self.assertAlmostEqual(fun(np.zeros(2)), 0.0)

    def test_trid(self):
        """Test the trid function."""
        fun = get_problem('trid', dimension=2)
        self.assertAlmostEqual(fun(np.array([2.0, 2.0])), -2.0)

    def test_perm(self):
        """Test the perm function."""
        fun = get_problem('perm', dimension=2)
        self.assertAlmostEqual(fun(np.array([1.0, 0.5])), 0.0)

    def test_zakharov(self):
        """Test the zakharov function."""
        fun = get_problem('zakharov', 2, -5, 10)
        self.assertAlmostEqual(fun(np.zeros(2)), 0.0)

    def test_dixon_price(self):
        """Test the dixon price function."""
        fun = get_problem('dixon_price', 2, -10, 10)
        solution = np.array([1.0, 0.70710678])
        self.assertAlmostEqual(fun(solution), 0.0)

    def test_powell(self):
        """Tests the powell function."""
        fun = get_problem('powell', dimension=2, lower=-4, upper=5)
        self.assertAlmostEqual(fun(np.zeros(2)), 0.0)

    def test_cosine_mixture(self):
        """Test the cosine mixture function."""
        fun = get_problem('cosine_mixture', dimension=2, lower=-1, upper=1)
        self.assertAlmostEqual(fun(np.zeros(2)), -0.2)


text:
import numpy as np

def func(a,b,c,d,e):
    return a+b+c+d+e

def wrapper(params,*args):
    return func(*params[:2],*args)

s=np.array([1,2,3,4,5])
print(func(*[],1,2,3,4,5))

max_stars_repo_name: ChendaHimalaya/qdt

max_issues_repo_name: ChendaHimalaya/qdt

max_forks_repo_name: ChendaHimalaya/qdt


text:
"""
This package extracts power and frequency characteristics of a signal.
"""
import numpy as np

def root_mean_square(audio, frame_length, hop_length):

    """
    Compute root-mean-square (RMS) value from scratch with the following formula,
    for each frame of the audio samples.
    """

    # Pad with the reflection of the signal so that the frames are centered
    # Padding is achieved by mirroring on the first and last values of the signal with frame_length // 2 samples
    signal = np.pad(audio, int(frame_length // 2), mode='reflect')

    rms = []

    for i in range(0, audio.shape[0], hop_length):

        rms_formula = np.sqrt(1 / frame_length * np.sum(signal[i:i+frame_length]**2))
        rms.append(rms_formula)

    return np.array(rms)

def spectral_centroid(audio, frame_length, hop_length, sr=44100):

    """
    As the name suggests, a spectral centroid is the location of the centre of mass of the spectrum.
    Since the audio files are digital signals and the spectral centroid is a measure that can be useful
    in the characterization of the spectrum of the audio file signal.
    """

    # Pad with the reflection of the signal so that the frames are centered
    # Padding is achieved by mirroring on the first and last values of the signal with frame_length // 2 samples
    signal = np.pad(audio, int(frame_length // 2), mode='reflect')

    centroid = []

    for i in range(0, audio.shape[0], hop_length):

        cent = signal[i:i+frame_length]

        # Compute the discrete Fourier Transform (DFT) with the efficient Fast Fourier Transform (FFT)
        magnitudes = np.abs(np.fft.fft(cent)) # magnitude of absolute (real) frequency values

        # Compute only the positive half of the DFT (i.e 1 + first half)
        mag = magnitudes[:int(1 + len(magnitudes) // 2)]

        # Compute the center frequencies of each bin
        freq = np.linspace(0, sr/2, int(1 + len(cent) // 2))

        # Return weighted mean of the frequencies present in the signal
        normalize_mag = mag / np.sum(mag)
        centroid.append(np.sum(freq * normalize_mag))

    return np.array(centroid)

def spectral_bandwidth(audio, frame_length, hop_length, sr=44100, p=2):

    """
    Bandwidth is the difference between the upper and lower frequencies in a continuous band of frequencies.
    As we know the signals oscillate about a point so if the point is the centroid of the signal then the sum of maximum
    deviation of the signal on both sides of the point can be considered as the bandwidth of the signal at that time frame.
    """

    # Pad with the reflection of the signal so that the frames are centered
    # Padding is achieved by mirroring on the first and last values of the signal with frame_length // 2 samples
    signal = np.pad(audio, int(frame_length // 2), mode='reflect')

    bandwidth = []

    for i in range(0, audio.shape[0], hop_length):

        frame = signal[i:i+frame_length]

        # Compute the discrete Fourier Transform (DFT) with the efficient Fast Fourier Transform (FFT)
        magnitudes = np.abs(np.fft.fft(frame)) # magnitude of absolute (real) frequency values

        # Compute only the positive half of the DFT (i.e 1 + first half)
        mag = magnitudes[:int(1 + len(magnitudes) // 2)]

        # Compute the center frequencies of each bin
        freq = np.linspace(0, sr/2, int(1 + len(frame) // 2))

        # Return weighted mean of the frequencies present in the signal
        normalize_mag = mag / np.sum(mag)
        centroid = np.sum(freq * normalize_mag)

        spectral_bandwidth = np.sum(normalize_mag * abs(freq - centroid) ** p) ** (1.0/p)
        bandwidth.append(spectral_bandwidth)

    return np.array(bandwidth)

max_stars_repo_path: Python Assignment 5/Part 1/features.py

max_issues_repo_path: Python Assignment 5/Part 1/features.py

max_forks_repo_path: Python Assignment 5/Part 1/features.py