text:
"""
Environments and wrappers for Sonic training.
"""

import gym
import numpy as np
import retro
from collections import deque
from gym import spaces
import cv2
import tensorflow as tf
import json

from baselines.common.atari_wrappers import WarpFrame, FrameStack
#import gym_remote.client as grc

def make_env(game=None, state=None, stack=False, scale_rew=True, allowbacktrace=False, custom=True):
    """
    Create an environment with some standard wrappers.
    """
    #env = grc.RemoteEnv('tmp/sock')
    #env = retro.make(game='SonicTheHedgehog-Genesis', state='GreenHillZone.Act1')
    #env = retro.make(game='StreetsOfRage2-Genesis', state='1Player.Axel.Level1')
    #env = retro.make(game='StreetFighterIISpecialChampionEdition-Genesis', state='Champion.Level1.RyuVsGuile')
    #env = retro.make(game='SuperMarioWorld-Snes', state='Bridges1')
    env = retro.make(game=game, state=state)
    #SuperMarioWorld-Snes ['Bridges1',
    env.seed(0)
    env = SonicDiscretizerV3(env)
    #env = StreetOfRage2Discretizer(env)
    #env = StreeFighter2Discretizer(env)
    if scale_rew:
        env = RewardScaler(env)
    env = WarpFrameRGB(env)
    if custom:
        env = CustomGym(env)
    if allowbacktrace:
        env = AllowBacktracking(env)
    if stack:
        env = FrameStack(env, 4)
    env = Controller_Gym(env)
    return env

class SonicDiscretizer(gym.ActionWrapper):
    """
    Wrap a gym-retro environment and make it use discrete
    actions for the Sonic game.
    """
    def __init__(self, env):
        super(SonicDiscretizer, self).__init__(env)
        buttons = ["B", "A", "MODE", "START", "UP", "DOWN", "LEFT", "RIGHT", "C", "Y", "X", "Z"]
        actions = [['LEFT'], ['RIGHT'], ['LEFT', 'DOWN'], ['RIGHT', 'DOWN'], ['DOWN'],
                   ['DOWN', 'B'], ['B'], [], ['LEFT', 'B'], ['RIGHT', 'B']]
        self._actions = []
        for action in actions:
            arr = np.array([False] * 12)
            for button in action:
                arr[buttons.index(button)] = True
            self._actions.append(arr)
        self.action_space = gym.spaces.Discrete(len(self._actions))

    def action(self, a): # pylint: disable=W0221
        return self._actions[a].copy()

class SonicDiscretizerV2(gym.ActionWrapper):
    """
    Wrap a gym-retro environment and make it use discrete
    actions for the Sonic game.
    """
    def __init__(self, env):
        super(SonicDiscretizerV2, self).__init__(env)
        buttons = ["B", "A", "MODE", "START", "UP", "DOWN", "LEFT", "RIGHT", "C", "Y", "X", "Z"]
        actions = [['LEFT'], ['RIGHT'], ['LEFT', 'DOWN'], ['RIGHT', 'DOWN'], ['DOWN'], ['DOWN', 'B'], ['B']]
        self._actions = []
        for action in actions:
            arr = np.array([False] * 12)
            for button in action:
                arr[buttons.index(button)] = True
            self._actions.append(arr)
        self.action_space = gym.spaces.Discrete(len(self._actions))

    def action(self, a): # pylint: disable=W0221
        return self._actions[a].copy()

class SonicDiscretizerV3(gym.ActionWrapper):
    """
    Wrap a gym-retro environment and make it use discrete
    actions for the Sonic game.
    """
    def __init__(self, env):
        super(SonicDiscretizerV3, self).__init__(env)
        buttons = ["B", "A", "MODE", "START", "UP", "DOWN", "LEFT", "RIGHT", "C", "Y", "X", "Z"]
        actions = [[], ['LEFT'], ['RIGHT'], ['B'], ['RIGHT', 'B'], ['DOWN'], ['DOWN','B'], ['RIGHT','B'], ['DOWN'], ['B'], []]
        self._actions = []
        for action in actions:
            arr = np.array([False] * 12)
            for button in action:
                arr[buttons.index(button)] = True
            self._actions.append(arr)
        self.action_space = gym.spaces.Discrete(len(self._actions))

    def action(self, a): # pylint: disable=W0221
        return self._actions[a].copy()

class StreetOfRage2Discretizer(gym.ActionWrapper):
    """
    Wrap a gym-retro environment and make it use discrete
    actions for the Sonic game.
    """
    def __init__(self, env):
        super(StreetOfRage2Discretizer, self).__init__(env)
        buttons = ["B", "A", "MODE", "START", "UP", "DOWN", "LEFT", "RIGHT", "C", "Y", "X", "Z"]
        actions = [['LEFT'], ['RIGHT'], ['UP'], ['DOWN'], ['LEFT', 'DOWN'], ['RIGHT', 'DOWN'], ['DOWN'],
                   ['DOWN', 'B'], ['B'], [], ['LEFT', 'B'], ['RIGHT', 'B']]
        self._actions = []
        for action in actions:
            arr = np.array([False] * 12)
            for button in action:
                arr[buttons.index(button)] = True
            self._actions.append(arr)
        self.action_space = gym.spaces.Discrete(len(self._actions))

    def action(self, a): # pylint: disable=W0221
        return self._actions[a].copy()

class StreeFighter2Discretizer(gym.ActionWrapper):
    """
    Wrap a gym-retro environment and make it use discrete
    actions for the Sonic game.
    """
    def __init__(self, env):
        super(StreeFighter2Discretizer, self).__init__(env)
        buttons = ["B", "A", "MODE", "START", "UP", "DOWN", "LEFT", "RIGHT", "C", "Y", "X", "Z"]
        actions = [['LEFT'], ['RIGHT'], ['UP'], ['DOWN'], ['LEFT', 'DOWN'], ['RIGHT', 'DOWN'],
                   ['DOWN', 'B'], ['B'], ['DOWN', 'A'], ['A'], ['DOWN', 'C'], ['C']]
        self._actions = []
        for action in actions:
            arr = np.array([False] * 12)
            for button in action:
                arr[buttons.index(button)] = True
            self._actions.append(arr)
        self.action_space = gym.spaces.Discrete(len(self._actions))

    def action(self, a): # pylint: disable=W0221
        return self._actions[a].copy()

class SuperMarioWorldDiscretizer(gym.ActionWrapper):
    """
    Wrap a gym-retro environment and make it use discrete
    actions for the Sonic game.
    """
    def __init__(self, env):
        super(SuperMarioWorldDiscretizer, self).__init__(env)
        buttons = ["B", "A", "MODE", "START", "UP", "DOWN", "LEFT", "RIGHT", "C", "Y", "X", "Z"]
        actions = [['LEFT'], ['RIGHT'], ['RIGHT', 'B'], ['B'], ['RIGHT', 'A'], ['A'],
                   ['RIGHT', 'C'], ['C'], ['RIGHT', 'Y'], ['Y'], ['RIGHT', 'X'], ['X'], ['RIGHT', 'Z'], ['Z']]
        self._actions = []
        for action in actions:
            arr = np.array([False] * 12)
            for button in action:
                arr[buttons.index(button)] = True
            self._actions.append(arr)
        self.action_space = gym.spaces.Discrete(len(self._actions))

    def action(self, a): # pylint: disable=W0221
        return self._actions[a].copy()

class RewardScaler(gym.RewardWrapper):
    """
    Bring rewards to a reasonable scale for PPO.
    This is incredibly important and effects performance
    drastically.
    """
    def reward(self, reward):
        return reward * 0.005

class AllowBacktracking(gym.Wrapper):
    """
    Use deltas in max(X) as the reward, rather than deltas
    in X. This way, agents are not discouraged too heavily
    from exploring backwards if there is no way to advance
    head-on in the level.
    """
    def __init__(self, env):
        super(AllowBacktracking, self).__init__(env)
        self._cur_x = 0
        self._max_x = 0

    def reset(self, **kwargs): # pylint: disable=E0202
        self._cur_x = 0
        self._max_x = 0
        return self.env.reset(**kwargs)

    def step(self, action): # pylint: disable=E0202
        obs, rew, done, info = self.env.step(action)
        self.env.render()
        self._cur_x += rew
        rew = max(0, self._cur_x - self._max_x)
        self._max_x = max(self._max_x, self._cur_x)
        return obs, rew, done, info

class CustomGym(gym.Wrapper):
    """
    add custom features
    """
    def __init__(self, env):
        super(CustomGym, self).__init__(env)

    def reset(self, **kwargs): # pylint: disable=E0202
        return self.env.reset(**kwargs)

    def step(self, action): # pylint: disable=E0202
        self.env.render()
        obs, rew, done, info = self.env.step(action)
        return obs, rew, done, info

class WarpFrameRGB(gym.ObservationWrapper):
    def __init__(self, env):
        """Warp frames to 84x84 as done in the Nature paper and later work."""
        gym.ObservationWrapper.__init__(self, env)
        self.width = 84
        self.height = 84
        self.observation_space = spaces.Box(low=0, high=255,
            shape=(self.height, self.width, 3), dtype=np.uint8)

    def observation(self, frame):
        frame = cv2.resize(frame, (self.width, self.height), interpolation=cv2.INTER_AREA)
        return frame

import numpy as np

import gym
from gym import error, spaces, utils
from gym.utils import seeding
import numpy as np
#from keyboard import KeyboardController, KeyLogger
#from mouse import MouseController, MouseLogger

class Controller_Gym(gym.Env):
    metadata = {
        "render.modes": ["human", "rgb_array"],
    }
    #Define Actions
    ACTION = [0,1]

    def __init__(self, env):
        self.controlling = True
        self.logging = True
        #self.mouse_logger = MouseLogger()
        #self.mouse_controller = MouseController()
        self.keyboard_logger = KeyLogger()
        #self.keyboard_controller = KeyboardController()
        self.env = env
        self.viewer = None
        self.info = None
        self.reward = None
        self.done = False
        self.state = None
        #self.action_dim = 3
        #self.state_dim = 109
        self.num_envs = 1
        self.num_envs_per_sub_batch = 1
        self.total_pips = []
        #self.player = self.env.player
        #self.pips = self.env.pips
        #self.starter = 0

        # forward or backward in each dimension
        #self.action_space = spaces.Discrete(3)
        self.action_space = self.env.action_space

        # observation is the x, y coordinate of the grid
        #low = np.zeros(0, dtype=int)
        #high =  np.array(1, dtype=int) - np.ones(len(self.maze_size), dtype=int)
        #self.observation_space = spaces.Box(low=-100000, high=100000, shape=(109,))
        self.observation_space = self.env.observation_space
        #print("obs")
        #print (self.observation_space)

        # initial condition
        #self.state = self.env.generate_number()
        self.steps_beyond_done = None

        # Simulation related variables.
        self.seed()
        #self.reset()

        # Just need to initialize the relevant attributes
        self.configure()

    def __del__(self):
        pass

    def configure(self, display=None):
        self.display = display

    def seed(self, seed=None):
        self.np_random, seed = seeding.np_random(seed)
        return [seed]

    def step(self, action):
        #self.state = self.env.generate_number()
        #self.env.display()
        #print(action)
        action = self.keyboard_logger.actions()
        #action = 1
        #self.placement = self.env.placement
        self.next_state, self.reward, self.done, info = self.env.step(action)
        #self.info = 0
        #print(self.reward)
        self.info = { 'pnl':1, 'nav':1, 'costs':1 }
        #self.next_state = self.next_state.tolist()
        #self.total_pips.append(self.pips)
        if self.done:
            pass
        return self.next_state, self.reward, self.done, info

    def reset(self):
        self.state = self.env.reset()
        #self.reward = np.array([reward])
        #self.state = self.state.tolist()
        #self.state = np.array([self.state])
        #self.steps_beyond_done = None
        self.done = False
        #self.done = np.array([self.done])
        return self.state

    def is_game_over(self):
        pass
        return

    def render(self, mode="human", close=False):
        self.env.render()
        #self.env.display()
        pass

        return

from pynput import keyboard
from pynput.keyboard import Key, Controller
moves =[False, False]

class KeyLogger():
    def __init__(self):
        self.moose = "9"
        self.listener = keyboard.Listener(on_press=self.on_press,on_release=self.on_release)
        self.listener.start()
        self.moves = [False, False, False, False, False]
        self.action = 0
    def on_press(self, key):
        try:
            print('alphanumeric key {0} pressed'.format(
                key.char))
            self.moose = key
            print(key)
            if key.char == '1':
                self.moves[1] = True
            if key.char == '0':
                self.moves[0] = True
            if key.char == '2':
                self.moves[2] = True
            if key.char == '3':
                self.moves[3] = True
            if key.char == '4':
                self.moves[4] = True

        except AttributeError:
            print('special key {0} pressed'.format(
                key))
        return

    def on_release(self, key):
        print('{0} released'.format(
            key))
        #self.moose = ''
        moose = key
        if key.char == "0":
            self.moves[0] = False
        if key.char == "1":
            self.moves[1] = False
        if key.char == "2":
            self.moves[2] = False
        if key.char == "3":
            self.moves[3] = False
        if key.char == "4":
            self.moves[4] = False
        if key == keyboard.Key.esc:
            # Stop listener
            return False
    def actions(self):
        if self.moves[3] == True and self.moves[2] == True:
            self.action = 4
        elif self.moves[1] == True:
            self.action = 1
        elif self.moves[2] == True:
            self.action = 2
        elif self.moves[3] == True:
            self.action = 3
        elif self.moves[3] == True:
            self.action = 3

        else:
            self.action = 0
        return self.action

class KeyboardController():
    def __init__(self):
        self.moose = "yeah"
        self.keyboard = Controller()

    def press(self, key):
        self.keyboard.press(Key.key)

    def release(self, key):
        self.keyboard.release(Key.key)

    def press_release(self, key):
        self.keyboard.press(Key.key)
        self.keyboard.release(Key.key)

    def typeing(self, string):
        self.keyboard.type(string)

from pynput import mouse
from pynput.mouse import Button, Controller

class MouseLogger():
    def __init__(self):
        self.test= "test"
        self.listener = mouse.Listener(on_move=self.on_move,on_click=self.on_click,on_scroll=self.on_scroll)
        self.listener()
    def on_move(self, x, y):
        print('Pointer moved to {0}'.format(
            (x, y)))

    def on_click(self, x, y, button, pressed):
        print('{0} at {1}'.format(
            'Pressed' if pressed else 'Released',
            (x, y)))
        if not pressed:
            # Stop listener
            return False

    def on_scroll(self, x, y, dx, dy):
        print('Scrolled {0} at {1}'.format(
            'down' if dy < 0 else 'up',
            (x, y)))

    def listener_block(self):
        with mouse.Listener(
            on_move=self.on_move,
            on_click=self.on_click,
            on_scroll=self.on_scroll) as self.listener:
            self.listener.join()

class MouseController():
    def __init__(self):
        pass
        self.mouse = Controller()

    def mouse_position(self):
        # Read pointer position
        print('The current pointer position is {0}'.format(
        self.mouse.position))
        return self.mouse.position

    def mouse_postiion_set(self, x, y):

        # Set pointer position
        self.mouse.position = (x, y)
        print('Now we have moved it to {0}'.format(
        self.mouse.position))

    def mouse_move(self, x, y):
        # Move pointer relative to current position
        self.mouse.move(x, y)

    def mouse_press(self, right=False):
        if right == True:
            self.mouse.press(Button.right)
        else:
            self.mouse.press(Button.left)

    def mouse_release(self, right=False):
        if right == True:
            self.mouse.release(Button.right)
        else:
            self.mouse.release(Button.left)

    def mouse_double_click(self, right=False):
        # Double click; this is different from pressing and releasing
        # twice on Mac OSX
        if right == 'right' or right == True:
            self.mouse.click(Button.right, 2)
        else:
            self.mouse.click(Button.right, 2)

    def mouse_scroll(self, x, y):
        # Scroll two steps down
        self.mouse.scroll(x, y)

max_stars_repo_name: adaptationio/Shaman-RL

max_issues_repo_name: adaptationio/Shaman-RL

max_forks_repo_name: adaptationio/Shaman-AI


text:
from __future__ import print_function #pylint bug workaround
import argparse
import pandas
import h5py
import numpy as np
from neuralnets.utils import one_hot_array, one_hot_index

from sklearn.model_selection import train_test_split

MAX_NUM_ROWS = 500000
SMILES_COL_NAME = "structure"
CATEGORIES_COL_NAME = "edge_categories"
MIN_BOUND_COL_NAME = "min_category"
MAX_BOUND_COL_NAME = "max_category"
MAX_WORD_LENGTH = 120
CHUNK_SIZE = 200

def get_arguments():
    parser = argparse.ArgumentParser(description='Prepare data for training')
    parser.add_argument('infile', type=str, help='Input file name')
    parser.add_argument('outfile', type=str, help='Output file name')
    parser.add_argument('--length', type=int, metavar='N', default = MAX_NUM_ROWS,
                        help='Maximum number of rows to include (randomly sampled).')
    parser.add_argument('--word_length', type=int, default = MAX_WORD_LENGTH,
                        help='Maximum length of input sequences.')
    parser.add_argument('--smiles_column', type=str, default = SMILES_COL_NAME,
                        help="Name of the column that contains the SMILES strings. Default: %s" % SMILES_COL_NAME)
    parser.add_argument("--categories_column", type=str, default = CATEGORIES_COL_NAME,
                        help="Name of the column that contains edge categories. Default: %s" % CATEGORIES_COL_NAME)
    parser.add_argument('--property_column', type=str,
                        help="Name of the column that contains the property values to predict. Default: None")

    return parser.parse_args()

# def chunk_iterator(dataset, chunk_size=CHUNK_SIZE):
#     chunk_indices = np.array_split(np.arange(len(dataset)),
#                                     len(dataset)/chunk_size)
#     for chunk_ixs in chunk_indices:
#         chunk = dataset[chunk_ixs]
#         yield (chunk_ixs, chunk)
#     raise StopIteration

def main():
    args = get_arguments()
    data = pandas.read_hdf(args.infile, 'table')
    print ("Number of SMILES strings: ", len(data))

    keys = data[args.smiles_column].map(len) < args.word_length + 1
    data = data[keys]

    keys = data[args.smiles_column].map(lambda x: list(x).count('9') == 0) == True
    data = data[keys]

    if args.length <= len(data):
        data = data.sample(n=args.length)

    data = data.reset_index(drop=True)

    ######################################################################################
    #SMILES strings
    ######################################################################################
    structures = data[args.smiles_column].map(lambda x: list(x.ljust(args.word_length)))

    charset = list(reduce(lambda x, y: set(y) | x, structures, set()))
    charset.sort()

    edge_categories = np.empty(dtype=int, shape=(0, args.word_length))

    train_idx, test_idx = map(np.array, train_test_split(structures.index, test_size=0.20))

    one_hot_encoded_fn = lambda row: map(lambda x: one_hot_array(x, len(charset)),
                                         one_hot_index(row, charset))

    h5f = h5py.File(args.outfile, 'w')
    h5f.create_dataset('charset', data=charset)

    h5f.create_dataset("data_train", data=np.array(map(one_hot_encoded_fn, structures[train_idx])), chunks=(CHUNK_SIZE, args.word_length, len(charset)))
    h5f.create_dataset("data_test", data=np.array(map(one_hot_encoded_fn, structures[test_idx])), chunks=(CHUNK_SIZE, args.word_length, len(charset)))

    ######################################################################################
    #edge categories
    ######################################################################################
    if args.categories_column in data.keys():
        max_category = 0
        for i, cat_str in enumerate(data[args.categories_column]):
            category_list = [int(c) for c in cat_str.split(" ")]
            max_category = max(max_category, max(category_list))

        edge_categories = np.append(edge_categories, np.full((len(data[args.categories_column]), args.word_length), max_category, dtype=int), axis = 0)
        for i, cat_str in enumerate(data[args.categories_column]):
            category_list = [int(c) for c in cat_str.split(" ")]
            for j, val in enumerate(category_list):
                edge_categories[i][j] = val

        #charset_cats = list(reduce(lambda x, y: set(y) | x, edge_categories, set()))
        #charset_cats.sort()
        charset_cats = range(max_category + 1)

        h5f.create_dataset('charset_cats', data=charset_cats)

        edge_categories_masks = np.empty(dtype=int, shape=(0, args.word_length, len(charset_cats)))
        if MIN_BOUND_COL_NAME in data.keys() and MAX_BOUND_COL_NAME in data.keys():

            edge_categories_masks = np.append(edge_categories_masks, np.zeros((len(data[args.categories_column]), args.word_length, len(charset_cats)), dtype=int), axis = 0)
            for i, (min_str, max_str) in enumerate(zip(data[MIN_BOUND_COL_NAME], data[MAX_BOUND_COL_NAME])):
                current_min = [int(c) for c in min_str.split(" ")]
                current_max = [int(c) for c in max_str.split(" ")]
                for j, _ in enumerate(zip(current_min, current_max)):
                    for k in range(current_min[j], current_max[j]):
                        edge_categories_masks[i][j][k] = 1
                for j in range(len(current_min), args.word_length):
                    edge_categories_masks[i][j][len(charset_cats) - 1] = 1

        one_hot_encoded_cats_fn = lambda row: map(lambda x: one_hot_array(x, len(charset_cats)),
                                                one_hot_index(row, charset_cats))

        if edge_categories.shape[0] > 0:
            h5f.create_dataset("categories_train", data=np.array(map(one_hot_encoded_cats_fn, edge_categories[train_idx])), chunks=(CHUNK_SIZE, args.word_length, len(charset_cats)))
            h5f.create_dataset("categories_test", data=np.array(map(one_hot_encoded_cats_fn, edge_categories[test_idx])), chunks=(CHUNK_SIZE, args.word_length, len(charset_cats)))

        if edge_categories_masks.shape[0] > 0:
            h5f.create_dataset("masks_train", data=edge_categories_masks[train_idx], chunks=(CHUNK_SIZE, args.word_length, len(charset_cats)))
            h5f.create_dataset("masks_test", data=edge_categories_masks[test_idx], chunks=(CHUNK_SIZE, args.word_length, len(charset_cats)))

    ######################################################################################
    #additional properties
    ######################################################################################
    if args.property_column:
        properties = data[args.property_column][keys]
        h5f.create_dataset('property_train', data=properties[train_idx])
        h5f.create_dataset('property_test', data=properties[test_idx])

    del data
    h5f.close()

    # def create_chunk_dataset(h5file, dataset_name, dataset, dataset_shape,
    #                          chunk_size=CHUNK_SIZE, apply_fn=None):
    #     new_data = h5file.create_dataset(dataset_name, dataset_shape,
    #                                      chunks=tuple([chunk_size]+list(dataset_shape[1:])))
    #     for (chunk_ixs, chunk) in chunk_iterator(dataset):
    #         if not apply_fn:
    #             new_data[chunk_ixs, ...] = chunk
    #         else:
    #             new_data[chunk_ixs, ...] = apply_fn(chunk)

    # create_chunk_dataset(h5f, 'data_train', train_idx,
    #                      (len(train_idx), args.word_length, len(charset)),
    #                      apply_fn=lambda ch: np.array(map(one_hot_encoded_fn,
    #                                                       structures[ch])))
    # create_chunk_dataset(h5f, 'data_test', test_idx,
    #                      (len(test_idx), args.word_length, len(charset)),
    #                      apply_fn=lambda ch: np.array(map(one_hot_encoded_fn,
    #                                                       structures[ch])))

    # if edge_categories.shape[0] > 0:
    #     create_chunk_dataset(h5f, 'categories_train', train_idx,
    #                          (len(train_idx), args.word_length, len(charset_cats)),
    #                          apply_fn=lambda c: np.array(map(one_hot_encoded_cats_fn, edge_categories[c].tolist())))

    #     create_chunk_dataset(h5f, 'categories_test', test_idx,
    #                          (len(test_idx), args.word_length, len(charset_cats)),
    #                          apply_fn=lambda c: np.array(map(one_hot_encoded_cats_fn, edge_categories[c].tolist())))

if __name__ == '__main__':
    main()

ext: py

reason: import numpy


text:
import sys, os, io
import numpy as np
import warnings
import pickle
from keras.models import load_model
import tensorflow as tf

def split_train_dataset(X_list, Y, test_ratio = 0.2):
    """
    Randomly splits the given dataset in train/test datasets according to the given ratio.
    [X_train, Y_train], [X_test, Y_test] = split_train_dataset(X, Y)
    """
    assert(len(X_list) > 0)

    num_samples = len(Y)
    shuffled_indices = np.random.permutation(num_samples)
    cut_point_idx = int(num_samples * (1 - test_ratio))
    train_indices, test_indices = shuffled_indices[:cut_point_idx], shuffled_indices[cut_point_idx:]

    X_train, X_test, Y_train, Y_test = [], [], [], []
    for feature_array in X_list:
        X_train.append(feature_array[train_indices])
        X_test.append(feature_array[test_indices])

    Y_train, Y_test = Y[train_indices], Y[test_indices]
    return [X_train, Y_train], [X_test, Y_test]

def deprecated(func):
    """This is a decorator which can be used to mark functions
    as deprecated. It will result in a warning being emmitted
    when the function is used."""
    def newFunc(*args, **kwargs):
        warnings.warn("Call to deprecated function %s." % func.__name__,
                      category=DeprecationWarning)
        return func(*args, **kwargs)
    newFunc.__name__ = func.__name__
    newFunc.__doc__ = func.__doc__
    newFunc.__dict__.update(func.__dict__)
    return newFunc

def find_in_sequence(predicate, seq):
    """
    Returns the index of the first element that matches the given condition, or -1 if none found.
    """
    idx = 0
    for elem in seq:
        if predicate(elem):
            return idx
        idx += 1
    return -1

def bin_scalar(dist, bins):
    for i in range(0, len(bins) - 1):
        if bins[i] <= dist < bins[i+1]:
            return i
    raise RuntimeWarning("Coudln't bin scalar '{}'.".format(dist))

def load_tokenizer(path):
    with open(path, 'rb') as f:
        tokenizer = pickle.load(f)
    return tokenizer

class MemCache(object):
    """
    Keeps an index/cache of the used tokenizers and models,
     preventing repeated loads.
    """

    def __init__(self, data_path):
        self.data_path = data_path
        self.tokenizers = dict()
        self.models = dict()
        self.model_graphs = dict()

    def get_tokenizer(self, key):
        if key not in self.tokenizers:
            self.tokenizers[key] = load_tokenizer(self.data_path + '/tokenizer.{}.pkl'.format(key))
        return self.tokenizers[key]

    def get_model(self, key):
        if key not in self.models:
            model = load_model(self.data_path + '/models/{}.h5'.format(key))
            model._make_predict_function()  # Initialize predict function in sync environment
            self.models[key] = model
            self.model_graphs[key] = tf.get_default_graph()
        return self.models[key]

    def predict(self, model_key, X):
        """
        See Keras' issue #2397.
        https://github.com/keras-team/keras/issues/2397
        """
        with self.model_graphs[model_key].as_default():
            return self.models[model_key].predict(X)

class Logger(object):
    def __init__(self, path):
        self.file = open(path, 'w')
        self.stdout = sys.stdout
        sys.stdout = self
    def __del__(self):
        self.close()
    def write(self, data):
        self.file.write(data)
        self.stdout.write(data)
    def flush(self):
        self.file.flush()
    def close(self):
        sys.stdout = self.stdout
        self.file.close()

max_stars_repo_name: AndreFCruz/coref-web-platform

max_issues_repo_name: AndreFCruz/coref-web-platform

max_forks_repo_name: AndreFCruz/coref-web-platform


text:
import numpy as np
import matplotlib.pyplot as plt
from constants import *
from mpl_toolkits import mplot3d

SMALL_SIZE = 15
MEDIUM_SIZE = 16
BIGGER_SIZE = 18

plt.rc('font', size=SMALL_SIZE)          # controls default text sizes
plt.rc('axes', titlesize=BIGGER_SIZE)     # fontsize of the axes title
plt.rc('axes', labelsize=BIGGER_SIZE)    # fontsize of the x and y labels
plt.rc('xtick', labelsize=SMALL_SIZE)    # fontsize of the tick labels
plt.rc('ytick', labelsize=SMALL_SIZE)    # fontsize of the tick labels
plt.rc('legend', fontsize=MEDIUM_SIZE)    # legend fontsize
plt.rc('figure', titlesize=BIGGER_SIZE)  # fontsize of the figure title

beam_widh_avg = [0.5107132213,	2.253482713,	3.332988013,	5.402778179,
7.232845341,	9.132660372,   11.409317,	12.69019839,
14.58297946,    15.16578467,	18.31799185,	19.19034909,
21.97136459,	22.74056641,	23.87716745]

beam_widh_max = [1.390314647,	4.829486405,	6.467216062,	7.945709593,
11.49356875,	13.03281282,	15.7254996,	17.52626206,
20.28306692,	20.65942554, 24.1170712,	24.47597979,
30.37138547,	29.84894391,	31.27158444]

time_delay = list(0.1*np.array(range(1, 16, 1)))

beam_widh_max_rad = [np.radians(x) for x in beam_widh_max]

power_wastage = [x**2 for x in beam_widh_max_rad]

directivity = [ 2.0/(1-np.cos(x)) for x in beam_widh_max_rad]

d_scale = (max(directivity) -  min(directivity))/2.0
t_scale =(max(time_delay) - min(time_delay))/2.0

# print(beam_widh_avg)
# print(time_delay)
#
# plt.scatter(time_delay, beam_widh_avg, color= 'blue', label='$ \\bar{ \\theta} _{min}  $')
# plt.scatter(time_delay, beam_widh_max, color= 'red', marker='^', label='$ max\{\\theta_{min}\} $')
# plt.xlabel("$ t_{delay} $")
# plt.ylabel("$ \\theta_{min} $")
# plt.legend(loc=2)
# plt.grid()

# Optimization

# fig = plt.figure()
# ax = plt.axes(projection='3d')

X_data= []
Y_data= []
Z_data= []
t_delay_reci = [1.0/x for x in time_delay]

for A in range(A_min, A_max):
    for B in range(B_min, B_max):
        f1 = np.multiply(0.01*A, directivity)
        f2 = np.multiply(B, time_delay)
        BUDGET = np.add(f1,f2)
        # print(BUDGET)
        X_data.append(0.1*A)
        plt.plot(time_delay[2:], f1[2:])
        plt.plot(time_delay[2:], f2[2:], color='red', linewidth=0.1)
        # Z_data.append( beam_widh_max[list(BUDGET).index( BUDGET[9] )] )

print(len(X_data))

# ax.scatter3D(X_data, Y_data, Z_data, c=Z_data, alpha=0.3)

# plt.plot(beam_widh_max, f1, color = 'blue')
# plt.plot(beam_widh_max, f2, color = 'red')

# plt.plot(beam_widh_max, BUDGET)

# print(beam_widh_max[list(BUDGET).index(min(BUDGET))])
plt.ylabel('$ A \\cdot D , B \\cdot t_{delay}$')
plt.xlabel('Time delay')
# plt.zlabel('$ A\\cdot D + B\\cdot t_{delay}  $')
# plt.plot(beam_widh_max, BUDGET, color= 'green')
plt.show()

max_stars_repo_name: monsij/Beam-align

max_issues_repo_name: monsij/Beam-align

max_forks_repo_name: monsij/Beam-align


text:
"""GloVe Embeddings + chars bi-LSTM + bi-LSTM + CRF

__author__ = "Guillaume Genthial"

01272020, jli34@UTH, update for transfer learning on ner tasks for I2B2-2010&SemEval-2015
"""

import functools
import json
import logging
from pathlib import Path
import os
import sys
import argparse

import numpy as np
import tensorflow as tf
from tf_metrics import precision, recall, f1

#DATADIR = '../../data/example'
# hyperparameters
parser = argparse.ArgumentParser(description='BiLSTM-CRF for NER task')

parser.add_argument('--gpu', type=str, default='0', help='available gpu number')

default_data_dir = 'data/ner_i2b2/source1/' #data/ner_i2b2/target1_transfer_learning/fold_1/input
parser.add_argument('--data_dir', type=str, default=default_data_dir +'/input', help='train raw_data source')
parser.add_argument('--model_dir', type=str, default=default_data_dir + '/model_ep150', help='model dir to save/load checkpoint')
parser.add_argument('--output_dir', type=str, default=default_data_dir + '/output_ep150', help='output dir')
parser.add_argument('--epoch', type=int, default=30, help='#epoch of training')
#parser.add_argument('--warm_start_dir', type=str, default='data/ner_i2b2/source1/model', help='#warm start for initial checkpoint')
parser.add_argument('--warm_start_dir', type=str, default=None, help='#warm start for initial checkpoint')

#parser.add_argument("--transformer",type=str,default="False",help="transer learning")
#parser.add_argument('--pretrain_embedding', type=str, default='random',help='use pretrained char embedding or init it randomly')

#parser.add_argument('--batch_size', type=int, default=32, help='#sample of each minibatch')

#parser.add_argument('--hidden_dim', type=int, default=300, help='#dim of hidden state')
#parser.add_argument('--optimizer', type=str, default='Adam', help='Adam/Adadelta/Adagrad/RMSProp/Momentum/SGD')
#parser.add_argument('--CRF', type=str2bool, default=True, help='use CRF at the top layer. if False, use Softmax')
#parser.add_argument('--lr', type=float, default=0.001, help='learning rate')
#parser.add_argument('--clip', type=float, default=5.0, help='gradient clipping')
#parser.add_argument('--dropout', type=float, default=0.6, help='dropout keep_prob')
#parser.add_argument('--update_embedding', type=str2bool, default=True, help='update embedding during training')
#parser.add_argument('--pretrain_embedding', type=str, default='emr-bert-embedding_mat.npy',help='use pretrained char embedding or init it randomly')
#parser.add_argument('--embedding_dim', type=int, default=768, help='random init char embedding_dim')
#parser.add_argument('--shuffle', type=str2bool, default=True, help='shuffle training raw_data before each epoch')
#parser.add_argument('--mode', type=str, default='train', help='train/test/demo')
#parser.add_argument('--demo_model', type=str, default='1565801182', help='model for test and demo')#random_char_300,1524919794
#parser.add_argument('--model', type=str, default='ner_i2b2_target1', help='model for train, test and demo')#random_char_300,1524919794
#parser.add_argument('--embedding_dir', type=str, default='word2vector', help='embedding files dir')
#parser.add_argument("--index",type=str,default='trainData/index.txt')
#parser.add_argument("--index",type=str,default='ner_i2b2_index.txt')

# Logging
#Path('results').mkdir(exist_ok=True)
#tf.logging.set_verbosity(logging.INFO)
#handlers = [
#    logging.FileHandler('results/main.log'),
#    logging.StreamHandler(sys.stdout)
#]
#logging.getLogger('tensorflow').handlers = handlers

def parse_fn(line_words, line_tags):
    # Encode in Bytes for TF
    words = [w.encode() for w in line_words.strip().split()]
    tags = [t.encode() for t in line_tags.strip().split()]
    assert len(words) == len(tags), "Words and tags lengths don't match"

    # Chars
    chars = [[c.encode() for c in w] for w in line_words.strip().split()]
    lengths = [len(c) for c in chars]
    max_len = max(lengths)
    chars = [c + [b'<pad>'] * (max_len - l) for c, l in zip(chars, lengths)]
    return ((words, len(words)), (chars, lengths)), tags

def generator_fn(words, tags):
    with Path(words).open('r') as f_words, Path(tags).open('r') as f_tags:
        for line_words, line_tags in zip(f_words, f_tags):
            yield parse_fn(line_words, line_tags)

def input_fn(words, tags, params=None, shuffle_and_repeat=False):
    params = params if params is not None else {}
    shapes = ((([None], ()),               # (words, nwords)
               ([None, None], [None])),    # (chars, nchars)
              [None])                      # tags
    types = (((tf.string, tf.int32),
              (tf.string, tf.int32)),
             tf.string)
    defaults = ((('<pad>', 0),
                 ('<pad>', 0)),
                'O')
    dataset = tf.data.Dataset.from_generator(
        functools.partial(generator_fn, words, tags),
        output_shapes=shapes, output_types=types)

    if shuffle_and_repeat:
        dataset = dataset.shuffle(params['buffer']).repeat(params['epochs'])

    dataset = (dataset
               .padded_batch(params.get('batch_size', 20), shapes, defaults)
               .prefetch(1))
    return dataset

def model_fn(features, labels, mode, params):
    # For serving features are a bit different
    if isinstance(features, dict):
        features = ((features['words'], features['nwords']),
                    (features['chars'], features['nchars']))

    # Read vocabs and inputs
    (words, nwords), (chars, nchars) = features
    dropout = params['dropout']
    training = (mode == tf.estimator.ModeKeys.TRAIN)
    vocab_words = tf.contrib.lookup.index_table_from_file(
        params['words'], num_oov_buckets=params['num_oov_buckets'])
    vocab_chars = tf.contrib.lookup.index_table_from_file(
        params['chars'], num_oov_buckets=params['num_oov_buckets'])
    with Path(params['tags']).open() as f:
        indices = [idx for idx, tag in enumerate(f) if tag.strip() != 'O']
        num_tags = len(indices) + 1
    with Path(params['chars']).open() as f:
        num_chars = sum(1 for _ in f) + params['num_oov_buckets']

    # Char Embeddings
    char_ids = vocab_chars.lookup(chars)
    variable = tf.get_variable(
        'chars_embeddings', [num_chars, params['dim_chars']], tf.float32)
    char_embeddings = tf.nn.embedding_lookup(variable, char_ids)
    char_embeddings = tf.layers.dropout(char_embeddings, rate=dropout,
                                        training=training)

    # Char LSTM
    dim_words = tf.shape(char_embeddings)[1]
    dim_chars = tf.shape(char_embeddings)[2]
    flat = tf.reshape(char_embeddings, [-1, dim_chars, params['dim_chars']])
    t = tf.transpose(flat, perm=[1, 0, 2])
    lstm_cell_fw = tf.contrib.rnn.LSTMBlockFusedCell(params['char_lstm_size'])
    lstm_cell_bw = tf.contrib.rnn.LSTMBlockFusedCell(params['char_lstm_size'])
    lstm_cell_bw = tf.contrib.rnn.TimeReversedFusedRNN(lstm_cell_bw)
    _, (_, output_fw) = lstm_cell_fw(t, dtype=tf.float32,
                                     sequence_length=tf.reshape(nchars, [-1]))
    _, (_, output_bw) = lstm_cell_bw(t, dtype=tf.float32,
                                     sequence_length=tf.reshape(nchars, [-1]))
    output = tf.concat([output_fw, output_bw], axis=-1)
    char_embeddings = tf.reshape(output, [-1, dim_words, 50])

    # Word Embeddings
    word_ids = vocab_words.lookup(words)
    glove = np.load(params['glove'])['embeddings']  # np.array
    variable = np.vstack([glove, [[0.] * params['dim']]])
    variable = tf.Variable(variable, dtype=tf.float32, trainable=False)
    word_embeddings = tf.nn.embedding_lookup(variable, word_ids)

    # Concatenate Word and Char Embeddings
    embeddings = tf.concat([word_embeddings, char_embeddings], axis=-1)
    embeddings = tf.layers.dropout(embeddings, rate=dropout, training=training)

    # LSTM
    t = tf.transpose(embeddings, perm=[1, 0, 2])  # Need time-major
    lstm_cell_fw = tf.contrib.rnn.LSTMBlockFusedCell(params['lstm_size'])
    lstm_cell_bw = tf.contrib.rnn.LSTMBlockFusedCell(params['lstm_size'])
    lstm_cell_bw = tf.contrib.rnn.TimeReversedFusedRNN(lstm_cell_bw)
    output_fw, _ = lstm_cell_fw(t, dtype=tf.float32, sequence_length=nwords)
    output_bw, _ = lstm_cell_bw(t, dtype=tf.float32, sequence_length=nwords)
    output = tf.concat([output_fw, output_bw], axis=-1)
    output = tf.transpose(output, perm=[1, 0, 2])
    output = tf.layers.dropout(output, rate=dropout, training=training)

    # CRF
    logits = tf.layers.dense(output, num_tags)
    crf_params = tf.get_variable("crf", [num_tags, num_tags], dtype=tf.float32)
    pred_ids, _ = tf.contrib.crf.crf_decode(logits, crf_params, nwords)

    if mode == tf.estimator.ModeKeys.PREDICT:
        # Predictions
        reverse_vocab_tags = tf.contrib.lookup.index_to_string_table_from_file(
            params['tags'])
        pred_strings = reverse_vocab_tags.lookup(tf.to_int64(pred_ids))
        predictions = {
            'pred_ids': pred_ids,
            'tags': pred_strings
        }
        return tf.estimator.EstimatorSpec(mode, predictions=predictions)
    else:
        # Loss
        vocab_tags = tf.contrib.lookup.index_table_from_file(params['tags'])
        tags = vocab_tags.lookup(labels)
        log_likelihood, _ = tf.contrib.crf.crf_log_likelihood(
            logits, tags, nwords, crf_params)
        loss = tf.reduce_mean(-log_likelihood)

        # Metrics
        weights = tf.sequence_mask(nwords)
        metrics = {
            'acc': tf.metrics.accuracy(tags, pred_ids, weights),
            'precision': precision(tags, pred_ids, num_tags, indices, weights),
            'recall': recall(tags, pred_ids, num_tags, indices, weights),
            'f1': f1(tags, pred_ids, num_tags, indices, weights),
        }
        for metric_name, op in metrics.items():
            tf.summary.scalar(metric_name, op[1])

        if mode == tf.estimator.ModeKeys.EVAL:
            return tf.estimator.EstimatorSpec(
                mode, loss=loss, eval_metric_ops=metrics)

        elif mode == tf.estimator.ModeKeys.TRAIN:
            train_op = tf.train.AdamOptimizer().minimize(
                loss, global_step=tf.train.get_or_create_global_step())
            return tf.estimator.EstimatorSpec(
                mode, loss=loss, train_op=train_op)

def write_predictions(estimator, input_words_file, input_tags_file, output_file):
    output_dir = os.path.dirname(output_file)
    Path(output_dir).mkdir(parents=True, exist_ok=True)
    with Path(output_file).open('wb') as f:
        test_inpf = functools.partial(input_fn, input_words_file, input_tags_file)
        golds_gen = generator_fn(input_words_file, input_tags_file)
        preds_gen = estimator.predict(test_inpf)
        for golds, preds in zip(golds_gen, preds_gen):
            ((words, _), (_, _)), tags = golds
            for word, tag, tag_pred in zip(words, tags, preds['tags']):
                f.write(b' '.join([word, tag, tag_pred]) + b'\n')
            f.write(b'\n')

if __name__ == '__main__':
    args = parser.parse_args()
    os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu

    #if len(sys.argv) > 1:
    #    DATADIR = sys.argv[1]
    #else:
    #    DATADIR = './data/ner_i2b2/source1/input'
    #print('DATADIR:{}'.format(DATADIR))

    # Logging
    log_dir = args.output_dir
    Path(log_dir).mkdir(exist_ok=True)
    tf.logging.set_verbosity(logging.INFO)
    handlers = [
        logging.FileHandler('{}/main.log'.format(log_dir)),
        logging.StreamHandler(sys.stdout)
    ]
    logging.getLogger('tensorflow').handlers = handlers

    # Params
    params = {
        'dim': 300,
        'dim_chars': 100,
        'dropout': 0.5,
        'num_oov_buckets': 1,
        'epochs': args.epoch, #25,
        'batch_size': 10, #20,
        'buffer': 10000, #15000,
        'char_lstm_size': 25,
        'lstm_size': 100,
        'words': str(Path(args.data_dir, 'vocab.words.txt')),
        'chars': str(Path(args.data_dir, 'vocab.chars.txt')),
        'tags': str(Path(args.data_dir, 'vocab.tags.txt')),
        'glove': str(Path(args.data_dir, 'glove.npz'))
    }
    with Path('{}/params.json'.format(args.data_dir)).open('w') as f:
        json.dump(params, f, indent=4, sort_keys=True)

    def fwords(name):
        return str(Path(args.data_dir, '{}.words.txt'.format(name)))

    def ftags(name):
        return str(Path(args.data_dir, '{}.tags.txt'.format(name)))

    # Estimator, train and evaluate
    train_inpf = functools.partial(input_fn, fwords('train'), ftags('train'),
                                   params, shuffle_and_repeat=True)
    eval_inpf = functools.partial(input_fn, fwords('testa'), ftags('testa'))

    cfg = tf.estimator.RunConfig(save_checkpoints_secs=120)

    warm_start_from = None
    if args.warm_start_dir:
        warm_start_from = tf.estimator.WarmStartSettings(
            ckpt_to_initialize_from=args.warm_start_dir,
            # NOTE: attempted with and without `vars_to_warm_start` set
            # vars_to_warm_start='^(?!.*(RMSProp|global_step))'
            vars_to_warm_start='^(?!.*(crf|dense/bias|dense/kernel))',
        )

    estimator = tf.estimator.Estimator(model_fn, '{}'.format(args.model_dir), cfg, params, warm_start_from=warm_start_from)
    Path(estimator.eval_dir()).mkdir(parents=True, exist_ok=True)
    #hook = tf.contrib.estimator.stop_if_no_increase_hook(
    hook = tf.estimator.experimental.stop_if_no_increase_hook(
        estimator, 'f1', 500, min_steps=8000, run_every_secs=120)
    train_spec = tf.estimator.TrainSpec(input_fn=train_inpf, hooks=[hook])
    eval_spec = tf.estimator.EvalSpec(input_fn=eval_inpf, throttle_secs=120)
    tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)

    #Path('{}/score'.format(args.output_dir)).mkdir(parents=True, exist_ok=True)
    for name in ['train', 'testa', 'testb']:
        input_words_file = fwords(name)
        input_tags_file = ftags(name)
        output_file = '{}/score/{}.preds.txt'.format(args.output_dir, name)
        write_predictions(estimator, input_words_file, input_tags_file, output_file)

    print('chars_lstm_lstm_crf train done!')


text:
import numpy as np

import torch
from tqdm import tqdm

from wavegrad.base import BaseModule
from wavegrad.nn import WaveGradNN

class WaveGrad(BaseModule):
    """
    WaveGrad diffusion process as described in WaveGrad paper
    (link: https://arxiv.org/pdf/2009.00713.pdf).
    Implementation adopted from `Denoising Diffusion Probabilistic Models`
    repository (link: https://github.com/hojonathanho/diffusion,
    paper: https://arxiv.org/pdf/2006.11239.pdf).
    """
    def __init__(self, preprocess_config, model_config):
        super(WaveGrad, self).__init__()
        # Setup noise schedule
        self.noise_schedule_is_set = False

        # Backbone neural network to model noise
        self.total_factor = np.product(model_config["wavegrad"]["factors"])
        assert self.total_factor == preprocess_config["preprocessing"]["stft"]["hop_length"], \
            """Total factor-product should be equal to the hop length of STFT."""
        self.nn = WaveGradNN(model_config)

    def set_new_noise_schedule(
        self,
        init=torch.linspace,
        init_kwargs={'steps': 50, 'start': 1e-6, 'end': 1e-2}
    ):
        """
        Sets sampling noise schedule. Authors in the paper showed
        that WaveGrad supports variable noise schedules during inference.
        Thanks to the continuous noise level conditioning.
        :param init (callable function, optional): function which initializes betas
        :param init_kwargs (dict, optional): dict of arguments to be pushed to `init` function.
            Should always contain the key `steps` corresponding to the number of iterations to be done by the model.
            This is done so because `torch.linspace` has this argument named as `steps`.
        """
        assert 'steps' in list(init_kwargs.keys()), \
            '`init_kwargs` should always contain the key `steps` corresponding to the number of iterations to be done by the model.'
        n_iter = init_kwargs['steps']

        betas = init(**init_kwargs)
        alphas = 1 - betas
        alphas_cumprod = alphas.cumprod(dim=0)
        alphas_cumprod_prev = torch.cat([torch.FloatTensor([1]), alphas_cumprod[:-1]])
        alphas_cumprod_prev_with_last = torch.cat([torch.FloatTensor([1]), alphas_cumprod])
        self.register_buffer('betas', betas)
        self.register_buffer('alphas', alphas)
        self.register_buffer('alphas_cumprod', alphas_cumprod)
        self.register_buffer('alphas_cumprod_prev', alphas_cumprod_prev)

        # Calculations for posterior q(y_n|y_0)
        sqrt_alphas_cumprod = alphas_cumprod.sqrt()
        # For WaveGrad special continuous noise level conditioning
        self.sqrt_alphas_cumprod_prev = alphas_cumprod_prev_with_last.sqrt().numpy()
        sqrt_recip_alphas_cumprod = (1 / alphas_cumprod).sqrt()
        sqrt_alphas_cumprod_m1 = (1 - alphas_cumprod).sqrt() * sqrt_recip_alphas_cumprod
        self.register_buffer('sqrt_alphas_cumprod', sqrt_alphas_cumprod)
        self.register_buffer('sqrt_recip_alphas_cumprod', sqrt_recip_alphas_cumprod)
        self.register_buffer('sqrt_alphas_cumprod_m1', sqrt_alphas_cumprod_m1)

        # Calculations for posterior q(y_{t-1} | y_t, y_0)
        posterior_variance = betas * (1 - alphas_cumprod_prev) / (1 - alphas_cumprod)
        posterior_variance = torch.stack([posterior_variance, torch.FloatTensor([1e-20] * n_iter)])
        posterior_log_variance_clipped = posterior_variance.max(dim=0).values.log()
        # below: log calculation clipped because the posterior variance is 0 at the beginning of the diffusion chain
        posterior_mean_coef1 = betas * alphas_cumprod_prev.sqrt() / (1 - alphas_cumprod)
        posterior_mean_coef2 = (1 - alphas_cumprod_prev) * alphas.sqrt() / (1 - alphas_cumprod)
        self.register_buffer('posterior_log_variance_clipped', posterior_log_variance_clipped)
        self.register_buffer('posterior_mean_coef1', posterior_mean_coef1)
        self.register_buffer('posterior_mean_coef2', posterior_mean_coef2)

        self.n_iter = n_iter
        self.noise_schedule_kwargs = {'init': init, 'init_kwargs': init_kwargs}
        self.noise_schedule_is_set = True

    def sample_continuous_noise_level(self, batch_size, device):
        """
        Samples continuous noise level sqrt(alpha_cumprod).
        This is what makes WaveGrad different from other Denoising Diffusion Probabilistic Models.
        """
        s = np.random.choice(range(1, self.n_iter + 1), size=batch_size)
        continuous_sqrt_alpha_cumprod = torch.FloatTensor(
            np.random.uniform(
                self.sqrt_alphas_cumprod_prev[s-1],
                self.sqrt_alphas_cumprod_prev[s],
                size=batch_size
            )
        ).to(device)
        return continuous_sqrt_alpha_cumprod.unsqueeze(-1)

    def q_sample(self, y_0, continuous_sqrt_alpha_cumprod=None, eps=None):
        """
        Efficiently computes diffusion version y_t from y_0 using a closed form expression:
            y_t = sqrt(alpha_cumprod)_t * y_0 + sqrt(1 - alpha_cumprod_t) * eps,
            where eps is sampled from a standard Gaussian.
        """
        batch_size = y_0.shape[0]
        continuous_sqrt_alpha_cumprod \
            = self.sample_continuous_noise_level(batch_size, device=y_0.device) \
                if isinstance(eps, type(None)) else continuous_sqrt_alpha_cumprod
        if isinstance(eps, type(None)):
            eps = torch.randn_like(y_0)
        # Closed form signal diffusion
        outputs = continuous_sqrt_alpha_cumprod * y_0 + (1 - continuous_sqrt_alpha_cumprod**2).sqrt() * eps
        return outputs

    def q_posterior(self, y_start, y, t):
        """
        Computes reverse (denoising) process posterior q(y_{t-1}|y_0, y_t, x)
        parameters: mean and variance.
        """
        posterior_mean = self.posterior_mean_coef1[t] * y_start + self.posterior_mean_coef2[t] * y
        posterior_log_variance_clipped = self.posterior_log_variance_clipped[t]
        return posterior_mean, posterior_log_variance_clipped

    def predict_start_from_noise(self, y, t, eps):
        """
        Computes y_0 from given y_t and reconstructed noise.
        Is needed to reconstruct the reverse (denoising)
        process posterior q(y_{t-1}|y_0, y_t, x).
        """
        return self.sqrt_recip_alphas_cumprod[t] * y - self.sqrt_alphas_cumprod_m1[t] * eps

    def p_mean_variance(self, texts, y, t, clip_denoised: bool):
        """
        Computes Gaussian transitions of Markov chain at step t
        for further computation of y_{t-1} given current state y_t and features.
        """
        batch_size = texts.shape[0]
        noise_level = torch.FloatTensor([self.sqrt_alphas_cumprod_prev[t+1]]).repeat(batch_size, 1).to(texts)
        eps_recon = self.nn(texts, y, noise_level)
        y_recon = self.predict_start_from_noise(y, t, eps_recon)

        if clip_denoised:
            y_recon.clamp_(-1.0, 1.0)

        model_mean, posterior_log_variance = self.q_posterior(y_start=y_recon, y=y, t=t)
        return model_mean, posterior_log_variance

    def compute_inverse_dynamics(self, texts, y, t, clip_denoised=True):
        """
        Computes reverse (denoising) process dynamics. Closely related to the idea of Langevin dynamics.
        :param texts (torch.Tensor): text features of shape [B, n_texts, T//hop_length]
        :param y (torch.Tensor): previous state from dynamics trajectory
        :param clip_denoised (bool, optional): clip signal to [-1, 1]
        :return (torch.Tensor): next state
        """
        model_mean, model_log_variance = self.p_mean_variance(texts, y, t, clip_denoised)
        eps = torch.randn_like(y) if t > 0 else torch.zeros_like(y)
        return model_mean + eps * (0.5 * model_log_variance).exp()

    def sample(self, texts, store_intermediate_states=False):
        """
        Samples speech waveform via progressive denoising of white noise with guidance of texts-epctrogram.
        :param texts (torch.Tensor): text features of shape [B, n_texts, T//hop_length]
        :param store_intermediate_states (bool, optional): whether to store dynamics trajectory or not
        :return ys (list of torch.Tensor) (if store_intermediate_states=True)
            or y_0 (torch.Tensor): predicted signals on every dynamics iteration of shape [B, T]
        """
        with torch.no_grad():
            device = next(self.parameters()).device
            batch_size, T = texts.shape[0], texts.shape[-1]
            ys = [torch.randn(batch_size, T*self.total_factor, dtype=torch.float32).to(device)]
            for t in tqdm(range(0, self.n_iter)[::-1]):
                y_t = self.compute_inverse_dynamics(texts, y=ys[-1], t=t)
                ys.append(y_t)
            return ys if store_intermediate_states else ys[-1]

    def compute_loss(self, texts, y_0):
        """
        Computes loss between GT Gaussian noise and reconstructed noise by model from diffusion process.
        :param texts (torch.Tensor): text features of shape [B, n_texts, T//hop_length]
        :param y_0 (torch.Tensor): GT speech signals
        :return loss (torch.Tensor): loss of diffusion model
        """
        self._verify_noise_schedule_existence()

        # Sample continuous noise level
        batch_size = y_0.shape[0]
        continuous_sqrt_alpha_cumprod \
            = self.sample_continuous_noise_level(batch_size, device=y_0.device)
        eps = torch.randn_like(y_0)

        # Diffuse the signal
        y_noisy = self.q_sample(y_0, continuous_sqrt_alpha_cumprod, eps)

        # Reconstruct the added noise
        eps_recon = self.nn(texts, y_noisy, continuous_sqrt_alpha_cumprod)
        loss = torch.nn.L1Loss()(eps_recon, eps)
        return loss

    def forward(self, texts, store_intermediate_states=False):
        """
        Generates speech from given text.
        :param texts (torch.Tensor): text tensor of shape [1, n_texts, T//hop_length]
        :param store_intermediate_states (bool, optional):
            flag to set return tensor to be a set of all states of denoising process
        """
        self._verify_noise_schedule_existence()

        return self.sample(
            texts, store_intermediate_states
        )

    def _verify_noise_schedule_existence(self):
        if not self.noise_schedule_is_set:
            raise RuntimeError(
                'No noise schedule is found. Specify your noise schedule '
                'by pushing arguments into `set_new_noise_schedule(...)` method. '
                'For example: '
                "`wavegrad.set_new_noise_level(init=torch.linspace, init_kwargs=\{'steps': 50, 'start': 1e-6, 'end': 1e-2\})`."
            )


text:
import random
from sympy.core.numbers import I
from sympy import symbols, Symbol, Rational, sqrt, Poly
from sympy.matrices import Matrix, eye, ones
from sympy.abc import x, y, z
from sympy.testing.pytest import raises
from sympy.matrices.matrices import MatrixDeterminant
from sympy.matrices.common import NonSquareMatrixError, _MinimalMatrix,  _CastableMatrix
from sympy.functions.combinatorial.factorials import factorial, subfactorial

class DeterminantOnlyMatrix(_MinimalMatrix, _CastableMatrix, MatrixDeterminant):
    pass

def test_determinant():

    for M in [Matrix(), Matrix([[1]])]:
        assert (
            M.det() ==
            M._eval_det_bareiss() ==
            M._eval_det_berkowitz() ==
            M._eval_det_lu() ==
            1)

    M = Matrix(( (-3,  2),
                 ( 8, -5) ))

    assert M.det(method="domain-ge") == -1
    assert M.det(method="bareiss") == -1
    assert M.det(method="berkowitz") == -1
    assert M.det(method="lu") == -1

    M = Matrix(( (x,   1),
                 (y, 2*y) ))

    assert M.det(method="domain-ge") == 2*x*y - y
    assert M.det(method="bareiss") == 2*x*y - y
    assert M.det(method="berkowitz") == 2*x*y - y
    assert M.det(method="lu") == 2*x*y - y

    M = Matrix(( (1, 1, 1),
                 (1, 2, 3),
                 (1, 3, 6) ))

    assert M.det(method="domain-ge") == 1
    assert M.det(method="bareiss") == 1
    assert M.det(method="berkowitz") == 1
    assert M.det(method="lu") == 1

    M = Matrix(( ( 3, -2,  0, 5),
                 (-2,  1, -2, 2),
                 ( 0, -2,  5, 0),
                 ( 5,  0,  3, 4) ))

    assert M.det(method="domain-ge") == -289
    assert M.det(method="bareiss") == -289
    assert M.det(method="berkowitz") == -289
    assert M.det(method="lu") == -289

    M = Matrix(( ( 1,  2,  3,  4),
                 ( 5,  6,  7,  8),
                 ( 9, 10, 11, 12),
                 (13, 14, 15, 16) ))

    assert M.det(method="domain-ge") == 0
    assert M.det(method="bareiss") == 0
    assert M.det(method="berkowitz") == 0
    assert M.det(method="lu") == 0

    M = Matrix(( (3, 2, 0, 0, 0),
                 (0, 3, 2, 0, 0),
                 (0, 0, 3, 2, 0),
                 (0, 0, 0, 3, 2),
                 (2, 0, 0, 0, 3) ))

    assert M.det(method="domain-ge") == 275
    assert M.det(method="bareiss") == 275
    assert M.det(method="berkowitz") == 275
    assert M.det(method="lu") == 275

    M = Matrix(( ( 3,  0,  0, 0),
                 (-2,  1,  0, 0),
                 ( 0, -2,  5, 0),
                 ( 5,  0,  3, 4) ))

    assert M.det(method="domain-ge") == 60
    assert M.det(method="bareiss") == 60
    assert M.det(method="berkowitz") == 60
    assert M.det(method="lu") == 60

    M = Matrix(( ( 1,  0,  0,  0),
                 ( 5,  0,  0,  0),
                 ( 9, 10, 11, 0),
                 (13, 14, 15, 16) ))

    assert M.det(method="domain-ge") == 0
    assert M.det(method="bareiss") == 0
    assert M.det(method="berkowitz") == 0
    assert M.det(method="lu") == 0

    M = Matrix(( (3, 2, 0, 0, 0),
                 (0, 3, 2, 0, 0),
                 (0, 0, 3, 2, 0),
                 (0, 0, 0, 3, 2),
                 (0, 0, 0, 0, 3) ))

    assert M.det(method="domain-ge") == 243
    assert M.det(method="bareiss") == 243
    assert M.det(method="berkowitz") == 243
    assert M.det(method="lu") == 243

    M = Matrix(( (1, 0,  1,  2, 12),
                 (2, 0,  1,  1,  4),
                 (2, 1,  1, -1,  3),
                 (3, 2, -1,  1,  8),
                 (1, 1,  1,  0,  6) ))

    assert M.det(method="domain-ge") == -55
    assert M.det(method="bareiss") == -55
    assert M.det(method="berkowitz") == -55
    assert M.det(method="lu") == -55

    M = Matrix(( (-5,  2,  3,  4,  5),
                 ( 1, -4,  3,  4,  5),
                 ( 1,  2, -3,  4,  5),
                 ( 1,  2,  3, -2,  5),
                 ( 1,  2,  3,  4, -1) ))

    assert M.det(method="domain-ge") == 11664
    assert M.det(method="bareiss") == 11664
    assert M.det(method="berkowitz") == 11664
    assert M.det(method="lu") == 11664

    M = Matrix(( ( 2,  7, -1, 3, 2),
                 ( 0,  0,  1, 0, 1),
                 (-2,  0,  7, 0, 2),
                 (-3, -2,  4, 5, 3),
                 ( 1,  0,  0, 0, 1) ))

    assert M.det(method="domain-ge") == 123
    assert M.det(method="bareiss") == 123
    assert M.det(method="berkowitz") == 123
    assert M.det(method="lu") == 123

    M = Matrix(( (x, y, z),
                 (1, 0, 0),
                 (y, z, x) ))

    assert M.det(method="domain-ge") == z**2 - x*y
    assert M.det(method="bareiss") == z**2 - x*y
    assert M.det(method="berkowitz") == z**2 - x*y
    assert M.det(method="lu") == z**2 - x*y

    # issue 13835
    a = symbols('a')
    M = lambda n: Matrix([[i + a*j for i in range(n)]
                          for j in range(n)])
    assert M(5).det() == 0
    assert M(6).det() == 0
    assert M(7).det() == 0

def test_issue_14517():
    M = Matrix([
        [   0, 10*I,    10*I,       0],
        [10*I,    0,       0,    10*I],
        [10*I,    0, 5 + 2*I,    10*I],
        [   0, 10*I,    10*I, 5 + 2*I]])
    ev = M.eigenvals()
    # test one random eigenvalue, the computation is a little slow
    test_ev = random.choice(list(ev.keys()))
    assert (M - test_ev*eye(4)).det() == 0

def test_legacy_det():
    # Minimal support for legacy keys for 'method' in det()
    # Partially copied from test_determinant()

    M = Matrix(( ( 3, -2,  0, 5),
                 (-2,  1, -2, 2),
                 ( 0, -2,  5, 0),
                 ( 5,  0,  3, 4) ))

    assert M.det(method="bareis") == -289
    assert M.det(method="det_lu") == -289
    assert M.det(method="det_LU") == -289

    M = Matrix(( (3, 2, 0, 0, 0),
                 (0, 3, 2, 0, 0),
                 (0, 0, 3, 2, 0),
                 (0, 0, 0, 3, 2),
                 (2, 0, 0, 0, 3) ))

    assert M.det(method="bareis") == 275
    assert M.det(method="det_lu") == 275
    assert M.det(method="Bareis") == 275

    M = Matrix(( (1, 0,  1,  2, 12),
                 (2, 0,  1,  1,  4),
                 (2, 1,  1, -1,  3),
                 (3, 2, -1,  1,  8),
                 (1, 1,  1,  0,  6) ))

    assert M.det(method="bareis") == -55
    assert M.det(method="det_lu") == -55
    assert M.det(method="BAREISS") == -55

    M = Matrix(( ( 3,  0,  0, 0),
                 (-2,  1,  0, 0),
                 ( 0, -2,  5, 0),
                 ( 5,  0,  3, 4) ))

    assert M.det(method="bareiss") == 60
    assert M.det(method="berkowitz") == 60
    assert M.det(method="lu") == 60

    M = Matrix(( ( 1,  0,  0,  0),
                 ( 5,  0,  0,  0),
                 ( 9, 10, 11, 0),
                 (13, 14, 15, 16) ))

    assert M.det(method="bareiss") == 0
    assert M.det(method="berkowitz") == 0
    assert M.det(method="lu") == 0

    M = Matrix(( (3, 2, 0, 0, 0),
                 (0, 3, 2, 0, 0),
                 (0, 0, 3, 2, 0),
                 (0, 0, 0, 3, 2),
                 (0, 0, 0, 0, 3) ))

    assert M.det(method="bareiss") == 243
    assert M.det(method="berkowitz") == 243
    assert M.det(method="lu") == 243

    M = Matrix(( (-5,  2,  3,  4,  5),
                 ( 1, -4,  3,  4,  5),
                 ( 1,  2, -3,  4,  5),
                 ( 1,  2,  3, -2,  5),
                 ( 1,  2,  3,  4, -1) ))

    assert M.det(method="bareis") == 11664
    assert M.det(method="det_lu") == 11664
    assert M.det(method="BERKOWITZ") == 11664

    M = Matrix(( ( 2,  7, -1, 3, 2),
                 ( 0,  0,  1, 0, 1),
                 (-2,  0,  7, 0, 2),
                 (-3, -2,  4, 5, 3),
                 ( 1,  0,  0, 0, 1) ))

    assert M.det(method="bareis") == 123
    assert M.det(method="det_lu") == 123
    assert M.det(method="LU") == 123

def eye_Determinant(n):
    return DeterminantOnlyMatrix(n, n, lambda i, j: int(i == j))

def zeros_Determinant(n):
    return DeterminantOnlyMatrix(n, n, lambda i, j: 0)

def test_det():
    a = DeterminantOnlyMatrix(2, 3, [1, 2, 3, 4, 5, 6])
    raises(NonSquareMatrixError, lambda: a.det())

    z = zeros_Determinant(2)
    ey = eye_Determinant(2)
    assert z.det() == 0
    assert ey.det() == 1

    x = Symbol('x')
    a = DeterminantOnlyMatrix(0, 0, [])
    b = DeterminantOnlyMatrix(1, 1, [5])
    c = DeterminantOnlyMatrix(2, 2, [1, 2, 3, 4])
    d = DeterminantOnlyMatrix(3, 3, [1, 2, 3, 4, 5, 6, 7, 8, 8])
    e = DeterminantOnlyMatrix(4, 4,
        [x, 1, 2, 3, 4, 5, 6, 7, 2, 9, 10, 11, 12, 13, 14, 14])
    from sympy.abc import i, j, k, l, m, n
    f = DeterminantOnlyMatrix(3, 3, [i, l, m, 0, j, n, 0, 0, k])
    g = DeterminantOnlyMatrix(3, 3, [i, 0, 0, l, j, 0, m, n, k])
    h = DeterminantOnlyMatrix(3, 3, [x**3, 0, 0, i, x**-1, 0, j, k, x**-2])
    # the method keyword for `det` doesn't kick in until 4x4 matrices,
    # so there is no need to test all methods on smaller ones

    assert a.det() == 1
    assert b.det() == 5
    assert c.det() == -2
    assert d.det() == 3
    assert e.det() == 4*x - 24
    assert e.det(method="domain-ge") == 4*x - 24
    assert e.det(method='bareiss') == 4*x - 24
    assert e.det(method='berkowitz') == 4*x - 24
    assert f.det() == i*j*k
    assert g.det() == i*j*k
    assert h.det() == 1
    raises(ValueError, lambda: e.det(iszerofunc="test"))

def test_permanent():
    M = Matrix([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
    assert M.per() == 450
    for i in range(1, 12):
        assert ones(i, i).per() == ones(i, i).T.per() == factorial(i)
        assert (ones(i, i)-eye(i)).per() == (ones(i, i)-eye(i)).T.per() == subfactorial(i)

    a1, a2, a3, a4, a5 = symbols('a_1 a_2 a_3 a_4 a_5')
    M = Matrix([a1, a2, a3, a4, a5])
    assert M.per() == M.T.per() == a1 + a2 + a3 + a4 + a5

def test_adjugate():
    x = Symbol('x')
    e = DeterminantOnlyMatrix(4, 4,
        [x, 1, 2, 3, 4, 5, 6, 7, 2, 9, 10, 11, 12, 13, 14, 14])

    adj = Matrix([
        [   4,         -8,         4,         0],
        [  76, -14*x - 68,  14*x - 8, -4*x + 24],
        [-122, 17*x + 142, -21*x + 4,  8*x - 48],
        [  48,  -4*x - 72,       8*x, -4*x + 24]])
    assert e.adjugate() == adj
    assert e.adjugate(method='bareiss') == adj
    assert e.adjugate(method='berkowitz') == adj

    a = DeterminantOnlyMatrix(2, 3, [1, 2, 3, 4, 5, 6])
    raises(NonSquareMatrixError, lambda: a.adjugate())

def test_util():
    R = Rational

    v1 = Matrix(1, 3, [1, 2, 3])
    v2 = Matrix(1, 3, [3, 4, 5])
    assert v1.norm() == sqrt(14)
    assert v1.project(v2) == Matrix(1, 3, [R(39)/25, R(52)/25, R(13)/5])
    assert Matrix.zeros(1, 2) == Matrix(1, 2, [0, 0])
    assert ones(1, 2) == Matrix(1, 2, [1, 1])
    assert v1.copy() == v1
    # cofactor
    assert eye(3) == eye(3).cofactor_matrix()
    test = Matrix([[1, 3, 2], [2, 6, 3], [2, 3, 6]])
    assert test.cofactor_matrix() == \
        Matrix([[27, -6, -6], [-12, 2, 3], [-3, 1, 0]])
    test = Matrix([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
    assert test.cofactor_matrix() == \
        Matrix([[-3, 6, -3], [6, -12, 6], [-3, 6, -3]])

def test_cofactor_and_minors():
    x = Symbol('x')
    e = DeterminantOnlyMatrix(4, 4,
        [x, 1, 2, 3, 4, 5, 6, 7, 2, 9, 10, 11, 12, 13, 14, 14])

    m = Matrix([
        [ x,  1,  3],
        [ 2,  9, 11],
        [12, 13, 14]])
    cm = Matrix([
        [ 4,         76,       -122,        48],
        [-8, -14*x - 68, 17*x + 142, -4*x - 72],
        [ 4,   14*x - 8,  -21*x + 4,       8*x],
        [ 0,  -4*x + 24,   8*x - 48, -4*x + 24]])
    sub = Matrix([
            [x, 1,  2],
            [4, 5,  6],
            [2, 9, 10]])

    assert e.minor_submatrix(1, 2) == m
    assert e.minor_submatrix(-1, -1) == sub
    assert e.minor(1, 2) == -17*x - 142
    assert e.cofactor(1, 2) == 17*x + 142
    assert e.cofactor_matrix() == cm
    assert e.cofactor_matrix(method="bareiss") == cm
    assert e.cofactor_matrix(method="berkowitz") == cm

    raises(ValueError, lambda: e.cofactor(4, 5))
    raises(ValueError, lambda: e.minor(4, 5))
    raises(ValueError, lambda: e.minor_submatrix(4, 5))

    a = DeterminantOnlyMatrix(2, 3, [1, 2, 3, 4, 5, 6])
    assert a.minor_submatrix(0, 0) == Matrix([[5, 6]])

    raises(ValueError, lambda:
        DeterminantOnlyMatrix(0, 0, []).minor_submatrix(0, 0))
    raises(NonSquareMatrixError, lambda: a.cofactor(0, 0))
    raises(NonSquareMatrixError, lambda: a.minor(0, 0))
    raises(NonSquareMatrixError, lambda: a.cofactor_matrix())

def test_charpoly():
    x, y = Symbol('x'), Symbol('y')
    z, t = Symbol('z'), Symbol('t')

    from sympy.abc import a,b,c

    m = DeterminantOnlyMatrix(3, 3, [1, 2, 3, 4, 5, 6, 7, 8, 9])

    assert eye_Determinant(3).charpoly(x) == Poly((x - 1)**3, x)
    assert eye_Determinant(3).charpoly(y) == Poly((y - 1)**3, y)
    assert m.charpoly() == Poly(x**3 - 15*x**2 - 18*x, x)
    raises(NonSquareMatrixError, lambda: Matrix([[1], [2]]).charpoly())
    n = DeterminantOnlyMatrix(4, 4, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
    assert n.charpoly() == Poly(x**4, x)

    n = DeterminantOnlyMatrix(4, 4, [45, 0, 0, 0, 0, 23, 0, 0, 0, 0, 87, 0, 0, 0, 0, 12])
    assert n.charpoly() == Poly(x**4 - 167*x**3 + 8811*x**2 - 173457*x + 1080540, x)

    n = DeterminantOnlyMatrix(3, 3, [x, 0, 0, a, y, 0, b, c, z])
    assert n.charpoly() == Poly(t**3 - (x+y+z)*t**2 + t*(x*y+y*z+x*z) - x*y*z , t)

max_stars_repo_name: ianmasc/sympy

max_issues_repo_name: ianmasc/sympy

max_forks_repo_name: ianmasc/sympy

reason: from sympy


text:
import pandas as pd
import numpy as np

lista_valores = range(4)
list_indices = list('CABD')
serie = pd.Series(lista_valores, index=list_indices)
sorted_index = serie.sort_index()
print(sorted_index)
sorted_value = serie.sort_values()
print(sorted_value)

list_rang = serie.rank()
print(list_rang)

serie2 = pd.Series(np.random.rand(10))
print(serie2)
list_rang2 = serie2.rank()
print(list_rang2)

max_stars_repo_name: FoxNeo/MyPythonProjects

max_issues_repo_name: FoxNeo/MyPythonProjects

max_forks_repo_name: FoxNeo/MyPythonProjects


text:
import numpy as np

import flimage

def test_get_fl():
    # make sure no errors when printing repr
    size = 50
    fl = np.repeat(np.linspace(0, 10, size), size)
    fl = fl.reshape(size, size)

    fli1 = flimage.FLImage(fl)

    # test flses
    assert np.allclose(fli1.fl, fl)

def test_get_amp_fl_nan():
    # make sure no errors when printing repr
    size = 50
    fl = np.repeat(np.linspace(0, 10, size), size)
    fl = fl.reshape(size, size)
    flnan = fl.copy()
    flnan[:4] = np.nan
    fli = flimage.FLImage(flnan)
    assert np.allclose(fli.fl, flnan, equal_nan=True)

def test_slice():
    size = 50
    fl = np.repeat(np.linspace(0, 10, size), size)
    fl = fl.reshape(size, size)

    fli = flimage.FLImage(fl)

    x = 25
    y = 10
    x_size = 25
    y_size = 5
    flic = fli[x:x + x_size, y:y + y_size]
    # simple sanity checks
    assert flic.shape == (x_size, y_size)
    # check bg_correction
    assert np.allclose(flic.fl, fl[x:x + x_size, y:y + y_size])

    # slice along x
    flic2 = fli[x:x + x_size]
    assert flic2.shape == (25, size)
    assert np.allclose(flic2.fl, fl[x:x + x_size])

    # index should not work
    try:
        fli[0]
    except ValueError:
        pass
    else:
        assert False, "simple indexing not supported"

if __name__ == "__main__":
    # Run all tests
    loc = locals()
    for key in list(loc.keys()):
        if key.startswith("test_") and hasattr(loc[key], "__call__"):
            loc[key]()

max_stars_repo_name: RI-imaging/flimage

max_issues_repo_name: RI-imaging/flimage

max_forks_repo_name: RI-imaging/flimage


text:
#File to draw the different shapes on an image
#Load the all the required libraries
import cv2
import matplotlib.pyplot as plt
import argparse
import numpy as np

ap = argparse.ArgumentParser()
# ap.add_argument('-i','--image',required=False,help='Pass the correct file path')
# args = vars(ap.parse_args())

#Load the images
# image = cv2.imread(args['image'])
canvas = np.zeros((300, 300, 3), dtype="uint8")
#Draw a line from top left to bottom right
green=(0,255,0)
cv2.line(canvas,(0,0),(300,300),green)
cv2.imshow('Canvas',canvas)
#Draw a 3 pixel thick red line from the top right to the bottom left
red = (0,0,255)
cv2.line(canvas,(300,0),(0,300),red,3)
cv2.imshow('Canvs with red line',canvas)

#Draw a green 60 x 60 pixel square , starting at 10,10 and ending at 60,60
cv2.rectangle(canvas,(10,10),(60,60),green)
cv2.imshow('Canvas having square',canvas)

#Draw another rectangle
cv2.rectangle(canvas,(50,200),(200,225),red,5)
cv2.imshow('Thick Rectangle of Red Color',canvas)

#Draw a solid rectangle , to draw a solid rectangle pass the argument as -1
blue = (255,0,0)
cv2.rectangle(canvas,(200,50),(225,125),blue,-1)
cv2.imshow('Canvas with filled blue rectangle',canvas)

#Circles
# reset our canvas and draw a white circle at the center of the canvas with
# increasing radii - from 25 pixels to 150 pixels
canvas = np.zeros((300, 300, 3), dtype="uint8")
(centerX, centerY) = (canvas.shape[1] // 2, canvas.shape[0] // 2)
white = (255, 255, 255)
for r in range(0, 175, 25):
	cv2.circle(canvas, (centerX, centerY), r, white)
# show our work of art
cv2.imshow("Canvas", canvas)
# let's go crazy and draw 25 random circles
for i in range(0, 25):
	# randomly generate a radius size between 5 and 200, generate a random
	# color, and then pick a random point on our canvas where the circle
	# will be drawn
	radius = np.random.randint(5, high=200)
	color = np.random.randint(0, high=256, size = (3,)).tolist()
	pt = np.random.randint(0, high=300, size = (2,))
	# draw our random circle
	cv2.circle(canvas, tuple(pt), radius, color, -1)
# Show our masterpiece
cv2.imshow("Canvas", canvas)

cv2.waitKey(0)
cv2.destroyAllWindows()

max_stars_repo_path: Module1/Module1.3 Drawing/Drawing.py

max_issues_repo_path: Module1/Module1.3 Drawing/Drawing.py

max_forks_repo_path: Module1/Module1.3 Drawing/Drawing.py


text:
import typing

import jax
import jax.numpy as jnp
import numpy as np
import scipy.integrate
import scipy.special

def linear_regression_st_risks(
        l2_lambda: typing.Union[float, jnp.ndarray],
        gamma: float,
        sigma_sq: float,
        epsilon_test: float
) -> typing.Tuple[typing.Union[float, jnp.ndarray], typing.Union[float, jnp.ndarray]]:
    orig_l2_lambda = l2_lambda
    if isinstance(orig_l2_lambda, float):
        l2_lambda = jnp.asarray([l2_lambda])

    if jnp.any(l2_lambda < 0):
        raise ValueError(f'Ridge parameters must be non-negative but are {l2_lambda}')
    if gamma <= 0:
        raise ValueError(f'd/n ratio must be positive but is {gamma}')
    if epsilon_test < 0:
        raise ValueError(f'Test attack epsilon must be non-negative but is {epsilon_test}')
    if sigma_sq < 0:
        raise ValueError(f'Noise variance must be non-negative but is {sigma_sq}')

    # Write m(z) = n(z) / z to avoid div by zero for lambda = 0
    def n(z: typing.Union[jnp.ndarray, float]) -> jnp.ndarray:
        t1 = 1.0 - gamma - z
        return (t1 - jnp.sqrt(jnp.square(t1) - 4.0 * gamma * z)) / (2.0 * gamma)
    n_prime = jax.vmap(jax.grad(n))

    bias = -l2_lambda * n_prime(-l2_lambda) - n(-l2_lambda)
    variance = sigma_sq * gamma * n_prime(-l2_lambda)
    proj_sq = bias + variance + 2.0 * n(-l2_lambda) + 1.0 - jnp.square(1.0 + n(-l2_lambda))

    standard_risk = bias + variance
    robust_risk = bias + variance \
        + 2.0 * jnp.sqrt(2.0 / jnp.pi) * epsilon_test * jnp.sqrt(proj_sq) * jnp.sqrt(standard_risk) \
        + jnp.square(epsilon_test) * proj_sq

    # Convert to float/numpy arrays
    if isinstance(orig_l2_lambda, float):
        standard_risk = float(standard_risk)
        robust_risk = float(robust_risk)

    return standard_risk, robust_risk

def logistic_regression_asymptotic_risks_linf(
        weight_norm_l2_orthogonal: float,
        weight_norm_l2_parallel: float,
        weight_norm_l1_projected: float,
        epsilon: float
) -> typing.Tuple[float, float]:
    # NB: This assumes that the ground truth has unit length!

    weight_norm_l2 = np.sqrt(np.square(weight_norm_l2_parallel) + np.square(weight_norm_l2_orthogonal))

    # If weight norm is zero, ground truth angle is ignored
    with np.errstate(divide='ignore'):
        ground_truth_cos_angle = weight_norm_l2_parallel / weight_norm_l2

    return logistic_regression_risks(
        weight_norm_l2=weight_norm_l2,
        ground_truth_cos_angle=ground_truth_cos_angle,
        weight_norm_lq_projected=weight_norm_l1_projected,
        epsilon=epsilon
    )

def logistic_regression_risks(
        weight_norm_l2: float,
        ground_truth_cos_angle: float,
        weight_norm_lq_projected: float,  # FIXME: This is more general, the name is a bit misleading
        epsilon: float
) -> typing.Tuple[float, float]:
    # Risks are simple if weights are zero
    if weight_norm_l2 == 0:
        true_std_risk = 0.5
        true_robust_risk = 1.0
    else:
        # Standard risk (1 - standard accuracy)
        true_std_risk = np.arccos(ground_truth_cos_angle) / np.pi

        # Robust risk
        assert ground_truth_cos_angle != 0  # else things break down, but this should not happen
        xi = float(epsilon * weight_norm_lq_projected / weight_norm_l2)
        t_term = np.abs(ground_truth_cos_angle) \
            / np.sqrt(1.0 - np.square(ground_truth_cos_angle))
        t_term = float(t_term)  # need a Python float for numerical integration later

        # Solve integral in risk numerically
        def integration_function(x: float) -> float:
            return 0.5 * scipy.special.erf(x / np.sqrt(2.0) * t_term) \
                   / np.sqrt(2.0 * np.pi) * np.exp(-0.5 * np.square(x))

        integral_approximation, *_ = scipy.integrate.quad(
            integration_function, a=0.0, b=xi
        )
        integral_term = 2.0 * integral_approximation
        true_robust_risk = true_std_risk + 0.5 * scipy.special.erf(xi / np.sqrt(2)) + integral_term

    return true_std_risk, true_robust_risk

def linear_regression_st_noiseless_risk_decomposition(
        l2_lambda: typing.Union[float, jnp.ndarray],
        gamma: float
) -> typing.Tuple[typing.Union[float, jnp.ndarray], typing.Union[float, jnp.ndarray]]:
    orig_l2_lambda = l2_lambda
    if isinstance(orig_l2_lambda, float):
        l2_lambda = jnp.asarray([l2_lambda])

    if jnp.any(l2_lambda < 0):
        raise ValueError(f'Ridge parameters must be non-negative but are {l2_lambda}')
    if gamma <= 0:
        raise ValueError(f'd/n ratio must be positive but is {gamma}')

    # Write m(z) = n(z) / z to avoid div by zero for lambda = 0
    def n(z: typing.Union[jnp.ndarray, float]) -> jnp.ndarray:
        t1 = 1.0 - gamma - z
        return (t1 - jnp.sqrt(jnp.square(t1) - 4.0 * gamma * z)) / (2.0 * gamma)
    n_prime = jax.vmap(jax.grad(n))

    bias = -l2_lambda * n_prime(-l2_lambda) - n(-l2_lambda)
    # Avoid negative norms due to numerical imprecision
    parallel_diff_norm_sq = jnp.maximum(0.0, jnp.square(1.0 + n(-l2_lambda)) - 2.0 * n(-l2_lambda) - 1.0)
    orthogonal_norm_sq = jnp.maximum(0.0, bias - parallel_diff_norm_sq)

    # Handle special case gamma = 1 and lambda = 0
    if gamma == 1:
        parallel_diff_norm_sq = jnp.where(l2_lambda == 0, 0.0, parallel_diff_norm_sq)
        orthogonal_norm_sq = jnp.where(l2_lambda == 0, 0.0, orthogonal_norm_sq)

    # Convert to float/numpy arrays
    if isinstance(orig_l2_lambda, float):
        parallel_diff_norm_sq = float(parallel_diff_norm_sq)
        orthogonal_norm_sq = float(orthogonal_norm_sq)
    return parallel_diff_norm_sq, orthogonal_norm_sq

reason: import numpy,import scipy,import jax


text:
import os
import numpy as np
import pandas as pd
from tensorflow.keras.utils import to_categorical
from sklearn import model_selection

class ACIC(object):
    def __init__(
            self,
            path,
            trial,
            center=False,
            exclude_population=False,
    ):
        self.trial = trial
        x_df, targets_df = load_data(
            path=path,
            trial=trial,
            subset='1'
        )
        train_dataset, test_dataset = train_test_split(
            x_df=x_df,
            targets_df=targets_df,
            trial=trial,
            test_size=0.3
        )
        self.train_data = get_trial(
            dataset=train_dataset
        )
        self.x_mean = self.train_data['x_cont'].mean(0, keepdims=True)
        self.x_std = self.train_data['x_cont'].std(0, keepdims=True) + 1e-7
        self.y_mean = self.train_data['y'].mean(dtype='float32')
        self.y_std = self.train_data['y'].std(dtype='float32') + 1e-7
        self.test_data = get_trial(
            dataset=test_dataset
        )
        self.dim_x_cont = self.train_data['x_cont'].shape[-1]
        self.dim_x_bin = self.train_data['x_bin'].shape[-1]
        self.dim_x = self.dim_x_cont + self.dim_x_bin

    def get_training_data(self):
        x, y, t = self.preprocess(self.train_data)
        examples_per_treatment = t.sum(0)
        return x, y, t, examples_per_treatment

    def get_test_data(self, test_set=True):
        _data = self.test_data if test_set else self.train_data
        x, _, _ = self.preprocess(_data)
        mu1 = _data['mu1'].astype('float32')
        mu0 = _data['mu0'].astype('float32')
        cate = mu1 - mu0
        return x, cate

    def get_subpop(self, test_set=True):
        _data = self.test_data if test_set else self.train_data
        return _data['ind_subpop']

    def get_t(self, test_set=True):
        _data = self.test_data if test_set else self.train_data
        return _data['t']

    def preprocess(self, dataset):
        x_cont = (dataset['x_cont'] - self.x_mean) / self.x_std
        x_bin = dataset['x_bin']
        x = np.hstack([x_cont, x_bin])
        y = (dataset['y'].astype('float32') - self.y_mean) / self.y_std
        t = dataset['t'].astype('float32')
        return x, y, t

def load_data(
        path,
        trial,
        subset='1'
):
    x_path = os.path.join(path, 'x.csv')
    targets_dir = os.path.join(path, str(trial + 1))
    targets_paths = os.listdir(targets_dir)
    targets_paths.sort()
    x_df = pd.read_csv(
        x_path
    )
    x_df['x_2'] = [ord(x) - 65 for x in x_df['x_2']]
    x_df['x_21'] = [ord(x) - 65 for x in x_df['x_21']]
    x_df['x_24'] = [ord(x) - 65 for x in x_df['x_24']]
    targets_df = pd.read_csv(
        os.path.join(
            targets_dir,
            targets_paths[0]
        )
    )
    return x_df, targets_df

def train_test_split(
        x_df,
        targets_df,
        trial,
        test_size=0.3,
):
    x_df_train, x_df_test, targets_df_train, targets_df_test = model_selection.train_test_split(
        x_df,
        targets_df,
        test_size=test_size,
        random_state=trial,
        shuffle=True
    )
    train_data = {
        'x': x_df_train,
        'targets': targets_df_train
    }
    test_data = {
        'x': x_df_test,
        'targets': targets_df_test
    }
    return train_data, test_data

def get_trial(
        dataset
):
    cat_feats = {'x_2': 6, 'x_21': 16, 'x_24': 5}
    bin_feats = ['x_17', 'x_22', 'x_38', 'x_51', 'x_54']
    cont_feats = []
    for i in range(1, 59):
        feat_id = 'x_{}'.format(i)
        if (feat_id not in bin_feats) and (feat_id not in cat_feats.keys()):
            cont_feats.append(feat_id)
    x_df = dataset['x']
    x_bin = x_df[bin_feats].to_numpy('float32')
    for k, v in cat_feats.items():
        f = dataset['x'][k].to_numpy()
        f = to_categorical(
            f,
            num_classes=v,
            dtype='float32'
        )
        x_bin = np.hstack([x_bin, f])
    x_cont = x_df[cont_feats].to_numpy('float32')
    targets_df = dataset['targets']
    t = targets_df['z'].to_numpy()
    y0 = targets_df['y0'].to_numpy()
    y1 = targets_df['y1'].to_numpy()
    y = np.zeros_like(t, 'float32')
    y[t > 0.5] = y1[t > 0.5]
    y[t < 0.5] = y0[t < 0.5]
    t_in = np.zeros((len(t), 2), 'float32')
    t_in[:, 0] = 1 - t
    t_in[:, 1] = t
    mu0 = targets_df['mu0'].to_numpy()
    mu1 = targets_df['mu1'].to_numpy()
    trial_data = {
        'x_cont': x_cont,
        'x_bin': x_bin,
        'y': y.astype('float32'),
        't': t_in.astype('float32'),
        'mu0': mu0.astype('float32'),
        'mu1': mu1.astype('float32')
    }
    return trial_data

max_stars_repo_name: HadarRosenwald/ucate

max_issues_repo_name: HadarRosenwald/ucate

max_forks_repo_name: HadarRosenwald/ucate