text:
import random
from datetime import datetime

import numpy as np
import pandas as pd
from matplotlib import pyplot as plt

from my_analytics.database import db_interface
from my_analytics.stats.colors import ColorEnum

plt.style.use("ggplot")

def plot_visit_counts():
    """Plots a random distribution of place visit counts"""
    places = db_interface.get_all_places()

    fig, ax = plt.subplots()

    places = places[:15]

    x, y, z = places.url, np.random.rand(len(places.url)), places.visit_count

    for _x, _y, _z in zip(x, y, z):
        plt.scatter(
            _x,
            _y,
            s=25 * _z,
            color=(random.uniform(0, 1), random.uniform(0, 1), random.uniform(0, 1)),
        )

    # add visit count annotation to plots
    for i, txt in enumerate(x):
        place = places[places.url == txt]  # get current 'place' series
        txt += f" ({int(place.visit_count)})"
        ax.annotate(txt, (x[i], y[i]))

    plt.title("Website Usage Visit Distributions")
    plt.xticks([])  # remove x ticks
    plt.yticks([])  # remove y ticks
    plt.show()

def plot_visit_times():
    """Scatters visit times"""
    visits = db_interface.get_all_visits()
    places = db_interface.get_all_places()
    visits["visit_month"] = visits["visit_date"].apply(
        lambda x: datetime.fromtimestamp(x / 1_000_000).strftime("%B")
    )  # get string representation of month
    visits["visit_hour"] = visits["visit_date"].apply(
        lambda x: int(datetime.fromtimestamp(x / 1_000_000).strftime("%H"))
    )  # get 24 hour value

    visit_counts = []
    for index, row in visits.iterrows():
        place = places[places.id == row.id]
        try:
            visit_counts.append(int(place.visit_count))
        except TypeError:  # some places have no visit_count
            visit_counts.append(0)

    x, y, z = visits.visit_hour, visit_counts, visits.visit_month

    for _x, _y, _z in zip(x, y, z):
        color = ColorEnum[_z].value  # index ColorEnum to get color for specific month
        plt.scatter(_x, _y, s=10 * _y, color=color)

    plt.title("Site Visit Time Distribution")
    plt.ylabel("Visit Count")
    plt.xlabel("Time (HH:MM)")
    plt.xticks(
        range(0, 24),
        (
            "00:00",
            "01:00",
            "02:00",
            "03:00",
            "04:00",
            "05:00",
            "06:00",
            "07:00",
            "08:00",
            "09:00",
            "10:00",
            "11:00",
            "12:00",
            "13:00",
            "14:00",
            "15:00",
            "16:00",
            "17:00",
            "18:00",
            "19:00",
            "20:00",
            "21:00",
            "22:00",
            "23:00",
        ),
    )  # set 24 hour x axis ticks
    plt.show()

max_stars_repo_name: AndrewRPorter/my-analytics

max_issues_repo_name: AndrewRPorter/my-analytics

max_forks_repo_name: AndrewRPorter/my-analytics


text:
from audioop import reverse
from dataclasses import dataclass, replace
from re import I
from typing import Callable, Dict, Sequence, Tuple, List
import numpy as np
from pandas import test
from scipy.stats import norm

from random import randrange
from numpy.polynomial.laguerre import lagval
from basis_fun import laguerre_polynomials
from price_model import SimulateGBM

from rich import print, pretty
pretty.install()

import matplotlib.pyplot as plt

from price_model import SimulateGBM

def laguerre_polynomials_ind(S, k):

    #  the first k terms of Laguerre Polynomials (k<=4)
#    x1 = np.exp(-S/2)
#    x2 = np.exp(-S/2) * (1 - S)
#    x3 = np.exp(-S/2) * (1 - 2*S + S**2/2)
#    x4 = np.exp(-S/2) * (1 - 3*S + 3* S**2/2 - S**3/6)

    u0 = np.ones(S.shape)
    x1 = 1 - S
    x2 = 1 - 2*S + S**2/2
    x3 = 1 - 3*S + 3*S**2/2 - S**3/6
    x4 = 1 - 4*S + 3*S**2 - 2*S**3/3 + S**4/24

    X  = [np.stack([u0, x1, x2]),
          np.stack([u0, x1, x2, x3]),
          np.stack([u0, x1, x2, x3, x4])]

    return X[k-2]

@dataclass(frozen=True)
class OptimalExerciseLSM:

    spot_price: float
    payoff: Callable[[float, float], float]
    expiry: float
    rate: float
    vol: float
    num_steps: int

    def GBMprice_training(self, num_paths_train) -> np.ndarray:

        return SimulateGBM(
            S0=self.spot_price,
            r=self.rate, sd=self.vol, T=self.expiry,
            paths=num_paths_train, steps=self.num_steps)

    def train_LSM(self, num_paths_train: int, K:float, k:int, training_data: np.ndarray):

            steps = int(self.num_steps)
            Stn = training_data
            #Stn = Stock_Matrix
            dt = self.expiry/steps
            cashFlow = np.zeros((num_paths_train, steps))
            cashFlow[:,steps - 1] = np.maximum(K-Stn[:,steps - 1], 0)

            cont_value = cashFlow

            decision = np.zeros((num_paths_train, steps))
            decision[:, self.num_steps - 1] = 1
            Beta_list = {}

            for i in reversed(range(steps-1)):

          # Find in the money paths
                in_the_money_n = np.where(K-Stn[:, i] > 0)[0]
                out_of_money_n = np.asarray(list(set(np.arange(num_paths_train)) - set(in_the_money_n)))

                X = laguerre_polynomials(Stn[in_the_money_n, i], k)
                Y = cashFlow[in_the_money_n, i + 1]/np.exp(self.rate*dt)

                A = np.dot(X.T, X)
                b = np.dot(X.T, Y)
                Beta = np.dot(np.linalg.pinv(A), b)
                try:
                    cont_value[out_of_money_n,i] =  cont_value[out_of_money_n, i + 1]/np.exp(r*dt)
                except:
                    pass

                decision[:, i] = np.where(np.maximum(K-Stn[:, i], 0)  - cont_value[:,i] >= 0, 1, 0)
                cashFlow[:, i] =  np.maximum(K-Stn[:, i], cont_value[:,i])

                Beta_list.update({i: Beta})

            return Beta_list

    def scoring_sim_data(self, num_paths_test: int) -> np.ndarray:
            return SimulateGBM(
            S0=self.spot_price,
            r=self.rate, sd=self.vol, T=self.expiry,
            paths=num_paths_test, steps=self.num_steps)

    def option_price(self,
        scoring_data: np.ndarray,
        Beta_list,
        k
        #func: FunctionApprox[Tuple[float, float]]
    ) -> float:
        num_paths: int = scoring_data.shape[0]
        prices: np.ndarray = np.zeros(num_paths)
        dt: float = self.expiry / self.num_steps

        #Beta_list.reverse()

        for i, path in enumerate(scoring_data):
            step: int = 0
            while step < self.num_steps:
                t: float = (step+1) * dt
                exercise_price: float = self.payoff(t, path[step])

                if exercise_price>0:
                    XX=laguerre_polynomials_ind(path[step],k)
                    continue_price: float = np.dot(XX, Beta_list[step])  \
                        if step < self.num_steps else 0.

                #continue_price: float = func.evaluate([(t, path[step])])[0] \
                #    if step < self.num_steps else 0.
                    step += 1
                    if exercise_price >= continue_price:
                        prices[i] = np.exp(-self.rate * t) * exercise_price
                        step = self.num_steps + 1
                else:
                    step += 1

        return np.average(prices)

K = 40
def payoff_func(_: float, s: float) -> float:
        return max(K - s, 0.)

LSMtest = OptimalExerciseLSM(spot_price=36,payoff=payoff_func,expiry=1, rate=0.06,
vol=0.2, num_steps=50)

GBMdata = LSMtest.GBMprice_training(num_paths_train=10000)
trained_lsm = LSMtest.train_LSM(num_paths_train=10000,K=40,k=4,
    training_data=GBMdata)
trained_lsm

GBMtest = LSMtest.scoring_sim_data(num_paths_test=1000)
GBMtest

price_of_option2 = LSMtest.option_price(scoring_data=GBMtest, Beta_list=trained_lsm,
                                        k=4)
price_of_option2

trained_lsm

LSMtest
GBMdata
GBMtest
trained_lsm

plt.figure(figsize=(15, 12))
plt.subplots_adjust(hspace=1)
plt.suptitle("Daily closing prices", fontsize=18, y=0.95)

TrainingDataType = Tuple[int, float, float]

def lsm_policy(S0, K, r, paths, sd, T, steps, Stock_Matrix,k, reduce_variance = True):
  steps = int(steps)
  Stn = Stock_Matrix
  #Stn = Stock_Matrix
  dt = T/steps
  cashFlow = np.zeros((paths, steps))
  cashFlow[:,steps - 1] = np.maximum(K-Stn[:,steps - 1], 0)

  cont_value = cashFlow

  decision = np.zeros((paths, steps))
  decision[:, steps - 1] = 1
  Beta_list = {}
  #discountFactor = np.tile(np.exp(-r*dt* np.arange(1,
  #                                    steps + 1, 1)), paths).reshape((paths, steps))

  for i in reversed(range(steps-1)):

          # Find in the money paths
          in_the_money_n = np.where(K-Stn[:, i] > 0)[0]
          out_of_money_n = np.asarray(list(set(np.arange(paths)) - set(in_the_money_n)))

          X = laguerre_polynomials(Stn[in_the_money_n, i], k)
          Y = cashFlow[in_the_money_n, i + 1]/np.exp(r*dt)

          A = np.dot(X.T, X)
          b = np.dot(X.T, Y)
          Beta = np.dot(np.linalg.pinv(A), b)

          cont_value[in_the_money_n,i] =  np.dot(X, Beta)

          #ax=plt.subplot(5, 2, i+1)
          #plt.subplot(211)
          #plt.plot(Stn[in_the_money_n, i], Y, "bs", Stn[in_the_money_n, i], cont_value[in_the_money_n,i],"rs")
          #plt.set_title('district_{}_change'.format(i))
          #plt.title('Time Step=' + str(i))
          #plt.xlabel("Price of the Underlying Asset (S)")
          #plt.ylabel("Continuation value , V(S)")
          #ax.set_ylim([0,40])
          #ax.set_xlim([10,40])
          try:
              cont_value[out_of_money_n,i] =  cont_value[out_of_money_n, i + 1]/np.exp(r*dt)
          except:
              pass

          decision[:, i] = np.where(np.maximum(K-Stn[:, i], 0)  - cont_value[:,i] >= 0, 1, 0)
          cashFlow[:, i] =  np.maximum(K-Stn[:, i], cont_value[:,i])

          Beta_list.update({i: Beta})

  return Beta_list

from price_model import SimulateGBM

S0_value = 36
r_value = 0.06
sd_value = 0.2
T_value = 1
paths_value = 10000
steps_value = 50

K_value = 40
k_value = 4

Stock_Matrix_GBM = SimulateGBM(S0=S0_value, r=r_value, sd=sd_value, T=T_value,
paths=paths_value,steps=steps_value, reduce_variance=False)

#Stock_Matrix_GBM_train = Stock_Matrix_GBM

#from rl_optexe import OptimalExerciseRL

strike = 40
def payoff_func(_: float, s: float) -> float:
        return max(strike - s, 0.)

#data_object = OptimalExerciseRL(spot_price=S0_value,payoff=payoff_func,expiry=1,
#rate=r_value, vol=sd_value, num_steps=steps_value)

#Stock_Matrix_GBM = data_object.scoring_sim_data(num_paths=paths_value)

#Stock_Matrix_GBM_train = Stock_Matrix_GBM[:,1:]
#Stock_Matrix_GBM_train

Beta_list1 = lsm_policy(S0=S0_value,
                                         K=K_value, r=r_value, paths=paths_value,
                                         sd=sd_value, T=T_value, steps=steps_value,
                                         Stock_Matrix=Stock_Matrix_GBM,
                                         k=k_value,
                                         reduce_variance=True)

#plt.show()
Beta_list1
price_of_option2 = LSMtest.option_price(scoring_data=GBMtest, Beta_list=Beta_list1,
                                        k=4)
def laguerre_polynomials_ind(S, k):

    #  the first k terms of Laguerre Polynomials (k<=4)
#    x1 = np.exp(-S/2)
#    x2 = np.exp(-S/2) * (1 - S)
#    x3 = np.exp(-S/2) * (1 - 2*S + S**2/2)
#    x4 = np.exp(-S/2) * (1 - 3*S + 3* S**2/2 - S**3/6)

    u0 = np.ones(S.shape)
    x1 = 1 - S
    x2 = 1 - 2*S + S**2/2
    x3 = 1 - 3*S + 3*S**2/2 - S**3/6
    x4 = 1 - 4*S + 3*S**2 - 2*S**3/3 + S**4/24

    X  = [np.stack([u0, x1, x2]),
          np.stack([u0, x1, x2, x3]),
          np.stack([u0, x1, x2, x3, x4])]

    return X[k-2]

# better regression equation of the

def option_price_lsm(
        scoring_data: np.ndarray,
        num_steps,
        payoff,
        rate,
        expiry,
        Beta_list,
        k
        #func: FunctionApprox[Tuple[float, float]]
    ) -> float:
        num_paths: int = scoring_data.shape[0]
        prices: np.ndarray = np.zeros(num_paths)
        dt: float = expiry / num_steps

        #Beta_list.reverse()

        for i, path in enumerate(scoring_data):
            step: int = 0
            while step <= num_steps:
                t: float = (step+1) * dt
                exercise_price: float = payoff(t, path[step])

                if exercise_price>0:
                    XX=laguerre_polynomials_ind(path[step],k)
                    continue_price: float = np.dot(XX, Beta_list[step])  \
                        if step < num_steps else 0.

                #continue_price: float = func.evaluate([(t, path[step])])[0] \
                #    if step < self.num_steps else 0.
                    step += 1
                    if exercise_price >= continue_price:
                        prices[i] = np.exp(-rate * t) * exercise_price
                        step = num_steps + 1
                else:
                    step += 1

        return np.average(prices)

#from rl_optexe import OptimalExerciseRL

strike = 40
def payoff_func(_: float, s: float) -> float:
        return max(strike - s, 0.)

#data_object = OptimalExerciseRL(spot_price=K_value,payoff=payoff_func,expiry=1,
#rate=r_value, vol=sd_value, num_steps=steps_value)

number_scoring_paths = 50000
#test_data = data_object.scoring_sim_data(num_paths=number_scoring_paths)
#Stock_Matrix_GBM_test=test_data[:,1:]

#Stock_Matrix_GBM_test
#test_data_1 = test_data[:,1:]

#test_data_1
#Beta_list
#steps_value_1 = steps_value-1

Stock_Matrix_GBM_test = SimulateGBM(S0=S0_value, r=r_value, sd=sd_value, T=T_value,
paths=10000,steps=steps_value, reduce_variance=False)

option_price_lsm(scoring_data=Stock_Matrix_GBM_test, num_steps=steps_value-1,
payoff=payoff_func, rate=r_value, expiry=T_value, Beta_list=trained_lsm, k=k_value)

price_of_option2 = LSMtest.option_price(scoring_data=GBMtest, Beta_list=Beta_list1,
                                        k=4)
Beta_list1
trained_lsm
stepss = 10
for i in reversed(range(10)):
    print(i)

testlist = [1,2,3]
reverse_list = reversed(testlist)
reverse_list
testlist

steps = 50
T =1
steps = int(steps)
r = 0.06
paths = 10
  #Stn = Stock_Matrix
dt = T/steps
  #cashFlow = np.zeros((paths, steps))
  #cashFlow[:,steps - 1] = np.maximum(K-Stn[:,steps - 1], 0)

 # cont_value = cashFlow

# decision = np.zeros((paths, steps))
# decision[:, steps - 1] = 1

discountFactor = np.tile(np.exp(-r*dt* np.arange(1,
                                      steps + 1, 1)), paths).reshape((paths, steps))
discountFactor

#np.exp(-rate * t)

import pandas as pd

price_of_option2 = LSMtest.option_price(scoring_data=GBMtest, Beta_list=Beta_list,
                                        k=4)
trained_lsm[48]
Beta_list1[48]

trained_lsm = LSMtest.train_LSM(num_paths_train=10000,K=40,k=4,
    training_data=Stock_Matrix_GBM)
trained_lsm[0]
Beta_list1[0]

########################################################
#######################################################
########################################################

plt.figure(figsize=(15, 12))
plt.subplots_adjust(hspace=0.7)
plt.suptitle("Expected Continutaion value in LSM", fontsize=16, y=0.98)

def lsm_policy(S0, K, r, paths, sd, T, steps, Stock_Matrix,k, reduce_variance = True):
  steps = int(steps)
  Stn = Stock_Matrix
  #Stn = Stock_Matrix
  dt = T/steps
  cashFlow = np.zeros((paths, steps))
  cashFlow[:,steps - 1] = np.maximum(K-Stn[:,steps - 1], 0)

  cont_value = cashFlow

  decision = np.zeros((paths, steps))
  decision[:, steps - 1] = 1
  Beta_list = {}
  #discountFactor = np.tile(np.exp(-r*dt* np.arange(1,
  #                                    steps + 1, 1)), paths).reshape((paths, steps))

  indext = 10
  for index, i in enumerate(reversed(range(steps-1))):

          # Find in the money paths
          in_the_money_n = np.where(K-Stn[:, i] > 0)[0]
          out_of_money_n = np.asarray(list(set(np.arange(paths)) - set(in_the_money_n)))

          X = laguerre_polynomials(Stn[in_the_money_n, i], k)
          Y = cashFlow[in_the_money_n, i + 1]/np.exp(r*dt)

          A = np.dot(X.T, X)
          b = np.dot(X.T, Y)
          Beta = np.dot(np.linalg.pinv(A), b)

          cont_value[in_the_money_n,i] =  np.dot(X, Beta)

          if i in np.arange(0,50,5):
            ax=plt.subplot(5, 2, indext)
          #plt.subplot(211)
            indext = indext -1
            xs, ys = zip(*sorted(zip(Stn[in_the_money_n, i], cont_value[in_the_money_n,i])))
            #plt.plot(Stn[in_the_money_n, i], Y, "bs", xs,
            #        ys,"r--",np.arange(25,41),40-np.arange(25,41), "g--",
            #        markersize=1, label="log10")
            l1 = ax.plot(Stn[in_the_money_n, i], Y, "bs", markersize=1,
            label = "continuation value")
            l2 = ax.plot(xs, ys,"r--",
            label = "Conditional Expectaion (Regresion)", linewidth=2)
            l3 = ax.plot(np.arange(20,41),40-np.arange(20,41), "g--",
            label = "Immediate Payoff", linewidth=2)

          #plt.set_title('district_{}_change'.format(i))
            plt.title('Time Step=' + str(i+1))
            plt.xlabel("Price of the Underlying Asset (S)")
            plt.ylabel("Continuation value , V(S)")
            ax.set_ylim([0,30])
            ax.set_xlim([10,40])
            if indext ==0:
                ax.legend(loc="lower left")
            #ax.legend([l1], "hda")
            #ax[1].legend(loc=(1.1, 0.5))

          try:
              cont_value[out_of_money_n,i] =  cont_value[out_of_money_n, i + 1]/np.exp(r*dt)
          except:
              pass

          decision[:, i] = np.where(np.maximum(K-Stn[:, i], 0)  - cont_value[:,i] >= 0, 1, 0)
          cashFlow[:, i] =  np.maximum(K-Stn[:, i], cont_value[:,i])

          Beta_list.update({i: Beta})

  return Beta_list

S0_value = 36
r_value = 0.06
sd_value = 0.4
T_value = 1
paths_value = 10000
steps_value = 50

K_value = 40
k_value = 4

Stock_Matrix_GBM = SimulateGBM(S0=S0_value, r=r_value, sd=sd_value, T=T_value,
paths=paths_value,steps=steps_value, reduce_variance=True)

np.shape(Stock_Matrix_GBM)[0]

Beta_list1 = lsm_policy(S0=S0_value,
                                         K=K_value, r=r_value, paths=paths_value,
                                         sd=sd_value, T=T_value, steps=steps_value,
                                         Stock_Matrix=Stock_Matrix_GBM,
                                         k=k_value,
                                         reduce_variance=True)
#plt.show()

# better regression equation of the

def option_price_lsm(
        scoring_data: np.ndarray,
        num_steps,
        payoff,
        rate,
        expiry,
        Beta_list,
        k
        #func: FunctionApprox[Tuple[float, float]]
    ) -> float:
        num_paths: int = scoring_data.shape[0]
        prices: np.ndarray = np.zeros(num_paths)
        dt: float = expiry / num_steps

        #Beta_list.reverse()

        for i, path in enumerate(scoring_data):
            step: int = 0
            while step <= num_steps:
                t: float = (step+1) * dt
                exercise_price: float = payoff(t, path[step])

                if exercise_price>0:
                    XX=laguerre_polynomials_ind(path[step],k)
                    continue_price: float = np.dot(XX, Beta_list[step])  \
                        if step < num_steps else 0.

                #continue_price: float = func.evaluate([(t, path[step])])[0] \
                #    if step < self.num_steps else 0.
                    step += 1
                    if exercise_price >= continue_price:
                        prices[i] = np.exp(-rate * t) * exercise_price
                        step = num_steps + 1
                else:
                    step += 1

        return np.average(prices)

strike = 40
def payoff_func(_: float, s: float) -> float:
        return max(strike - s, 0.)

Stock_Matrix_GBM_test = SimulateGBM(S0=S0_value, r=r_value, sd=sd_value, T=T_value,
paths=10000,steps=steps_value, reduce_variance=True)

option_price_lsm(scoring_data=Stock_Matrix_GBM_test, num_steps=steps_value-1,
payoff=payoff_func, rate=r_value, expiry=T_value, Beta_list=Beta_list1, k=k_value)

import matplotlib.pyplot as plt

prices_xaxis: np.ndarray = np.arange(40.0)

for i,step in enumerate(np.arange(0,48,8)):

    continue_price_list = []

    for price in prices_xaxis:

        XX=laguerre_polynomials_ind(price,k=4)
        continue_price: float = np.dot(XX, Beta_list1[step])
        continue_price_list.append((continue_price))
    ax=plt.subplot(3, 2, i+1)
    #plt.subplot(211)
    plt.plot(prices_xaxis,continue_price_list,"bs")
    #plt.set_title('district_{}_change'.format(i))
    plt.title('Time Step=' + str(i))
    plt.xlabel("Price of the Underlying Asset (S)")
    plt.ylabel("Continuation value , V(S)")

plt.show()
for i, j in enumerate(np.arange(0,48,8)):
    print(i)

ext: py

max_stars_repo_name: Peymankor/Tutorial-Derivative-Pricing

max_issues_repo_name: Peymankor/Tutorial-Derivative-Pricing

max_forks_repo_name: Peymankor/Tutorial-Derivative-Pricing

reason: import numpy,from numpy,from scipy


text:
import numpy as np
import os, sys
import torch
SGADHOME='/home/skvara/work/counterfactual_ad/sgad'
sys.path.append(SGADHOME)
from sgad.utils import datadir
from sgad.utils import save_resize_img

# train wildlife mnist
datapath = datadir("raw_datasets/wildlife_MNIST")
data = np.load(os.path.join(datapath, "data.npy"))
labels = np.load(os.path.join(datapath, "labels.npy"))

rng = np.random.default_rng()
x = []
for i in range(10):
    x.append(rng.choice(data[labels == i], 10))

x = np.concatenate(x)
save_resize_img(torch.tensor(x), os.path.join(datapath, "train.png"), 10)

# test wildlife mnist
datapath = datadir("raw_datasets/wildlife_MNIST")
data = np.load(os.path.join(datapath, "data_test.npy"))
labels = np.load(os.path.join(datapath, "labels_test.npy"))

rng = np.random.default_rng()
x = []
for i in range(10):
    x.append(rng.choice(data[np.array([l[0] for l in labels]) == i], 10))
x = np.concatenate(x)
save_resize_img(torch.tensor(x), os.path.join(datapath, "test_digit.png"), 10)

x = []
for i in range(10):
    x.append(rng.choice(data[np.array([l[1] for l in labels]) == i], 10))
x = np.concatenate(x)
save_resize_img(torch.tensor(x), os.path.join(datapath, "test_background.png"), 10)

x = []
for i in range(10):
    x.append(rng.choice(data[np.array([l[2] for l in labels]) == i], 10))
x = np.concatenate(x)
save_resize_img(torch.tensor(x), os.path.join(datapath, "test_texture.png"), 10)

reason: import numpy


text:
# -*- coding: utf-8 -*-
#
#    Copyright 2017 Jose A. Jimenez-Berni
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.
#
import datetime as dt
import numpy
import pytz
import pandas as pd

__author__ = "Jose A. Jimenez-Berni  <berni@ias.csic.es>"
__version__ = "0.1"
__date__ = "July 2017"

def get_day_of_year(datetime):
    """
    Calculate day of the year (DOY) for a given datetime object

    Parameters
    ----------
    datetime : Union[pd.DatetimeIndex, dt.datetime]
        datetime representing the date

    Returns
    -------
    object : day of the year as days after first of January

    """
    if isinstance(datetime, pd.DatetimeIndex):
        return datetime.dayofyear
    else:
        year_start = dt.datetime(datetime.year, 1, 1, tzinfo=datetime.tzinfo)
        delta = (datetime - year_start)
        return delta.days

def get_solar_time(longitude_deg, datetime):
    """
    Calculates the decimal hour for a given location and datetime

    Parameters
    ----------
    longitude_deg : Union[ndarray, float]
        Geographic longitude [degrees]
    datetime : Union[pd.DatetimeIndex, dt.datetime]
        Datetime with the date and time. If the datetime has not timezone it is assumed that is UTC

    Returns
    -------
    Union[ndarray, float]: decimal hour of the day

    """
    if isinstance(datetime, pd.DatetimeIndex):
        if datetime.tzinfo is None:
            # If the datetime has not timezone it is assumed that is UTC
            utc_datetime = datetime.tz_localize(pytz.utc)
        else:
            utc_datetime = datetime.tz_convert(pytz.utc)
        doy = get_day_of_year(datetime)
        solar_h = ((utc_datetime.hour * 60.0) + utc_datetime.minute + utc_datetime.second / 60.0 + (
            4.0 * longitude_deg) + equation_of_time(doy)) / 60.0
        return (solar_h + 24.0) % 24.0

    if datetime.tzinfo is None:
        # If the datetime has not timezone it is assumed that is UTC
        utc_datetime = datetime.replace(tzinfo=pytz.utc)
    else:
        utc_datetime = datetime.astimezone(pytz.utc)
    doy = get_day_of_year(datetime)
    return ((utc_datetime.hour * 60.0) + utc_datetime.minute + utc_datetime.second / 60.0 + (
        4.0 * longitude_deg) + equation_of_time(doy)) / 60.0

def equation_of_time(day_of_the_year):
    """

    Parameters
    ----------
    day_of_the_year

    Returns
    -------

    """
    b = (2.0 * numpy.pi / 364.0) * (day_of_the_year - 81.0)
    return (9.87 * numpy.sin(2.0 * b)) - (7.53 * numpy.cos(b)) - (1.5 * numpy.sin(b))

max_stars_repo_name: OpenAgriTech/opencroplib

max_issues_repo_name: OpenAgriTech/opencroplib

max_forks_repo_name: OpenAgriTech/opencroplib


text:
from math import log2
import numpy as np

class Node():
    def __init__(self, value=None, attribute_name="root", attribute_index=None, branches=None):
        """
        This class implements a tree structure with multiple branches at each node.
        If self.branches is an empty list, this is a leaf node and what is contained in
        self.value is the predicted class.

        The defaults for this are for a root node in the tree.

        Arguments:
            branches (list): List of Node classes. Used to traverse the tree. In a
                binary decision tree, the length of this list is either 2 (for left and
                right branches) or 0 (at a leaf node).
            attribute_name (str): Contains name of attribute that the tree splits the data
                on. Used for visualization (see `DecisionTree.visualize`).
            attribute_index (float): Contains the  index of the feature vector for the
                given attribute. Should match with self.attribute_name.
            value (number): Contains the value that data should be compared to along the
                given attribute.
        """
        self.branches = [] if branches is None else branches
        self.attribute_name = attribute_name
        self.attribute_index = attribute_index
        self.value = value

def branch_features(attribute_col, features, targets):
    branch1 = np.empty((0,features.shape[1]))
    branch2 = np.empty((0,features.shape[1]))
    target1 = np.empty(0)
    target2 = np.empty(0)

    for i in range(0, len(attribute_col)):
        if attribute_col[i]:
            branch1 = np.vstack((branch1,features[i]))
            target1 = np.append(target1,targets[i])
        else:
            branch2 = np.vstack((branch2,features[i]))
            target2 = np.append(target2,targets[i])
    return branch1, branch2, target1, target2

def ID3(features, targets, attribute_names):
    node = Node()
    if np.all(targets == 1):
        node.attribute_name = "leaf"
        node.value = 1
        return node
    elif np.all(targets == 0):
        node.attribute_name = "leaf"
        node.value = 0
        return node
    elif np.size(features) == 0:
        labels, counts = np.unique(targets, return_counts=True)
        maxindex = counts.argmax()
        node.value = labels[maxindex]
        node.attribute_name = "leaf"
        return node
    else:
        Att = best_attribute(features, targets)
        node.attribute_index = Att
        node.attribute_name = attribute_names[Att]
        new_features = np.delete(features, Att, 1)
        new_attribute_names = np.delete(attribute_names, Att)
        branch1, branch2, target1, target2 = branch_features(features[:,Att],new_features,targets)
        node1 = Node()
        node2 = Node()
        labels, counts = np.unique(targets, return_counts=True)
        maxindex = counts.argmax()
        most_common = labels[maxindex]

        if np.size(target1) == 0:
            node1.attribute_name = "leaf"
            node1.value = most_common
        else:
            node1 = ID3(branch1, target1, new_attribute_names)

        if np.size(target2) == 0:
            node2.attribute_name = "leaf"
            node2.value = most_common
        else:
            node2 = ID3(branch2, target2, new_attribute_names)
        node.branches = [node1, node2]

    return node

class DecisionTree():
    def __init__(self, attribute_names):
        """
        TODO: Implement this class.

        This class implements a binary decision tree learner for examples with
        categorical attributes. Use the ID3 algorithm for implementing the Decision
        Tree: https://en.wikipedia.org/wiki/ID3_algorithm

        A decision tree is a machine learning model that fits data with a tree
        structure. Each branching point along the tree marks a decision (e.g.
        today is sunny or today is not sunny). Data is filtered by the value of
        each attribute to the next level of the tree. At the next level, the process
        starts again with the remaining attributes, recursing on the filtered data.

        Which attributes to split on at each point in the tree are decided by the
        information gain of a specific attribute.

        Here, you will implement a binary decision tree that uses the ID3 algorithm.
        Your decision tree will be contained in `self.tree`, which consists of
        nested Node classes (see above).

        Args:
            attribute_names (list): list of strings containing the attribute names for
                each feature (e.g. chocolatey, good_grades, etc.)

        """
        self.attribute_names = attribute_names
        self.tree = None

    def _check_input(self, features):
        if features.shape[1] != len(self.attribute_names):
            raise ValueError(
                "Number of features and number of attribute names must match!"
            )

    def fit(self, features, targets):
        """
        Takes in the features as a numpy array and fits a decision tree to the targets.

        Args:
            features (np.array): numpy array of size NxF containing features, where N is
                number of examples and F is number of features.
            targets (np.array): numpy array containing class labels for each of the N
                examples.
        Output:
            VOID: It should update self.tree with a built decision tree.
        """
        self._check_input(features)

        self.tree = ID3(features, targets, self.attribute_names)
        self.visualize()

    def predict_onerow(self, node, example):
        if len(node.branches) == 0:
            return node.value
        else:
            attribute_index = self.attribute_names.index(node.attribute_name)
            if example[attribute_index] == 1:
                return self.predict_onerow(node.branches[0],example)
            else:
                return self.predict_onerow(node.branches[1],example)

    def predict(self, features):
        """
        Takes in features as a numpy array and predicts classes for each point using
        the trained model.

        Args:
            features (np.array): numpy array of size NxF containing features, where N is
                number of examples and F is number of features.
        Outputs:
            predictions (np.array): numpy array of size N array which has the predicitons
            for the input data.
        """
        self._check_input(features)
        # for each row of features,
        predictions = np.empty((0))
        for i in features:
            predictions = np.append(predictions,self.predict_onerow(self.tree, i))
        return predictions

    def _visualize_helper(self, tree, level):
        """
        Helper function for visualize a decision tree at a given level of recursion.
        """
        tab_level = "  " * level
        val = tree.value if tree.value is not None else 0
        print("%d: %s%s == %f" % (level, tab_level, tree.attribute_name, val))

    def visualize(self, branch=None, level=0):
        """
        Visualization of a decision tree. Implemented for you to check your work and to
        use as an example of how to use the given classes to implement your decision
        tree.
        """
        if not branch:
            branch = self.tree
        self._visualize_helper(branch, level)

        for branch in branch.branches:
            self.visualize(branch, level+1)

def best_attribute(features, targets):
    best_ig = -1
    best_att = -1
    for attribute_index in range(0,features.shape[1]):
        cur_ig = information_gain(features, attribute_index, targets)
        if cur_ig > best_ig:
            best_ig = cur_ig
            best_att = attribute_index
    return best_att

def binary_split(attributes,targets):
    pos_set = []
    neg_set = []
    for i in range(0, len(attributes)):
        if attributes[i]:
            pos_set.append(targets[i])
        else:
            neg_set.append(targets[i])
    return pos_set, neg_set

def entropy(fraction):
    if fraction == 0:
        return 0
    else:
        return -1*fraction*log2(fraction)

def information_gain(features, attribute_index, targets):

    pos_parent = np.sum(targets)/np.size(targets)
    neg_parent = 1- pos_parent
    parent_entropy =  entropy(pos_parent) + entropy(neg_parent)

    attribute = features[:,attribute_index]
    child_entropy = 0
    pos_set, neg_set = binary_split(attribute, targets)
    pos_set_pos = np.sum(pos_set)/np.size(pos_set)
    pos_set_neg = 1-pos_set_pos
    neg_set_pos = np.sum(neg_set)/np.size(neg_set)
    neg_set_neg = 1- neg_set_pos
    pos_set_weight = np.size(pos_set)/np.size(targets)
    neg_set_weight = np.size(neg_set)/np.size(targets)

    child_entropy_pos = entropy(pos_set_pos)*pos_set_weight + entropy(pos_set_neg)*pos_set_weight
    child_entropy_neg = entropy(neg_set_pos)*neg_set_weight + entropy(neg_set_neg)*neg_set_weight
    child_entropy = child_entropy_neg+ child_entropy_pos

    info_gain = parent_entropy-child_entropy

    return info_gain

if __name__ == '__main__':
    # construct a fake tree
    attribute_names = ['larry', 'curly', 'moe']
    decision_tree = DecisionTree(attribute_names=attribute_names)
    while len(attribute_names) > 0:
        attribute_name = attribute_names[0]
        if not decision_tree.tree:
            decision_tree.tree = Node(
                attribute_name=attribute_name,
                attribute_index=decision_tree.attribute_names.index(attribute_name),
                value=0,
                branches=[]
            )
        else:
            decision_tree.tree.branches.append(
                Node(
                    attribute_name=attribute_name,
                    attribute_index=decision_tree.attribute_names.index(attribute_name),
                    value=0,
                    branches=[]
                )
            )
        attribute_names.remove(attribute_name)
    decision_tree.visualize()


text:
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import axes3d

# Create data
N = 60
g1 = (0.6 + 0.6 * np.random.rand(N), np.random.rand(N),0.4+0.1*np.random.rand(N))
g2 = (0.4+0.3 * np.random.rand(N), 0.5*np.random.rand(N),0.1*np.random.rand(N))
g3 = (0.3*np.random.rand(N),0.3*np.random.rand(N),0.3*np.random.rand(N))

data = (g1, g2, g3)
colors = ("red", "green", "blue")
groups = ("coffee", "tea", "water")

# Create plot
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1, axisbg="1.0")
ax = fig.gca(projection='3d')

for data, color, group in zip(data, colors, groups):
    x, y, z = data
    ax.scatter(x, y, z, alpha=0.8, c=color, edgecolors='none', s=30, label=group)

plt.title('Matplot 3d scatter plot')
plt.legend(loc=2)
plt.show()


text:
# -*- coding: utf-8 -*-

from __future__ import (absolute_import, unicode_literals)

"""
==================================
RegulonDB Parsing and Web Services
==================================

:Authors:
    Moritz Emanuel Beber
:Date:
    2012-06-03
:Copyright:
    Copyright(c) 2012 Jacobs University of Bremen. All rights reserved.
:File:
    regulondb.py
"""

import logging
import re

import numpy as np
from builtins import (zip, dict)

from operator import attrgetter
from lxml import etree
from .. import miscellaneous as misc
from ..errors import PyOrganismError
from .generic import open_file, read_tabular
from ..regulation import elements as elem
from ..regulation.networks import GRN, TRN

LOGGER = logging.getLogger(__name__)
LOGGER.addHandler(misc.NullHandler())

BPATTERN = re.compile(r"b|B\d{4}")
FUNCTIONS = {"repressor": -1, "activator": 1, "unknown": 0, "dual": 2,
        "-": -1, "+": 1, "?": 0, "+-": 2}
NAPs = frozenset(["ECK120011186", "ECK120011224", "ECK120011229", "ECK120011235",
        "ECK120011294", "ECK120011345", "ECK120011383"])
RELEASE = {
    "5.0": "March 16, 2006",
    "5.1": "May 12, 2006",
    "5.2": "June 8, 2006",
    "5.5": "October 30, 2006",
    "5.6": "January 15, 2007",
    "5.7": "June 1, 2007",
    "5.8": "September 17, 2007",
    "6.0": "January 15, 2008",
    "6.1": "April 15, 2008",
    "6.2": "July 10, 2008",
    "6.3": "February 10, 2009",
    "6.4": "August 10, 2009",
    "6.7": "March 24, 2010",
    "6.8": "August 18, 2010",
    "7.0": "January 26, 2011",
    "7.2": "May 6, 2011",
    "7.3": "November 1, 2011",
    "7.4": "March 29, 2012",
    "7.5": "August 29, 2012",
    "8.0": "October 2, 2012",
    "8.1": "December 17, 2012",
    "8.2": "April 22, 2013",
    "8.3": "July 29, 2013",
    "8.5": "November 28, 2013",
    "8.6": "April 11, 2014",
}

def iter_rowset_soup(file_handle):
    raise NotImplementedError()
    from bs4 import BeautifulSoup
    soup = BeautifulSoup(file_handle, "lxml")
    for row in soup.rowset.find_all(name="row", recursive=False):
        yield dict()

def iter_rowset_lxml(filename):
    parser = etree.HTMLParser(encoding="latin1")
    tree = etree.parse(filename, parser)
    for el in tree.iter(tag="row"):
        yield dict((child.tag, etree.tostring(child, method="text",
                encoding=unicode)) for child in el.iterchildren())

def iter_rowset_flat_file(filename, header, sep="\t", comment="#"):
    for row in read_tabular(filename, sep=sep, comment=comment):
        yield dict((name, row[i]) for (i, name) in enumerate(header))

FILE_PARSERS = {
        ".xml": iter_rowset_lxml
}

def read_genes(filename, version="default", sep="\t", comment="#",
        encoding=None, mode="rb", **kw_args):
    """
    Retrieve the gene locations from a RegulonDB flat file and construct the
    GPN using the `proximity_threshold`.

    Parameters
    ----------
    filename: str
        The path to the file containing the interaction information.
    gene_identifier: str (optional)
        Identifiers for the genes used in the network. Can be any of:
            * 'name' for the gene name in that organism
            * 'blattner' for the Blattner number
            * 'regulondb' for the unique identifier assigned by RegulonDB
    sep: str (optional)
        The column separator; not likely to change.
    comment: str (optional)
        The sign denoting a comment line.
    encoding: str (optional)
        The file encoding.
    mode: str (optional)
        The mode used to open a file, should always be read binary.
    """
    genes = list()
    bpattern = BPATTERN
    kw_args["mode"] = mode
    kw_args["encoding"] = encoding
    with open_file(filename, **kw_args) as (file_h, ext):
        iter_rowset = FILE_PARSERS.get(ext, iter_rowset_flat_file)
        for row in iter_rowset(file_h):
            start = row["gene_posleft"]
            if not start:
                start = None
            end = row["gene_posright"]
            if not end:
                end = None
            gc = row["gc_content"]
            if not gc:
                gc = None
            name = row["gene_name"]
            if name and bpattern.match(name):
                bnumber = name
            else:
                bnumber = None

            gene = elem.Gene(unique_id=row["gene_id"],
                name_space=version,
                name=name,
                bnumber=bnumber,
                position_start=start,
                position_end=end,
                strand=row["gene_strand"],
                gc_content=gc)
            genes.append(gene)
    return genes

def update_gene_synonyms(filename, version="default", sep="\t", comment="#",
        encoding=None, mode="rb", **kw_args):
    bpattern = BPATTERN
    kw_args["mode"] = mode
    kw_args["encoding"] = encoding
    with open_file(filename, **kw_args) as (file_h, ext):
        iter_rowset = FILE_PARSERS.get(ext, iter_rowset_flat_file)
        for row in iter_rowset(file_h):
            obj_id = row["object_id"]
            if not elem.Gene.has_key(obj_id, version):
                continue
            gene = elem.Gene[obj_id, version]
            synonym = row["object_synonym_name"]
            if synonym is not None:
                if bpattern.match(synonym):
                    if gene.bnumber or "obsolete" in synonym:
                        gene.synonyms.add(synonym)
                    else:
                        gene.bnumber = synonym
                else:
                    gene.synonyms.add(synonym)

def update_gene_external(filename, version="default", sep="\t", comment="#",
        encoding=None, mode="rb", **kw_args):
    bpattern = BPATTERN
    kw_args["mode"] = mode
    kw_args["encoding"] = encoding
    with open_file(filename, **kw_args) as (file_h, ext):
        iter_rowset = FILE_PARSERS.get(ext, iter_rowset_flat_file)
        for row in iter_rowset(file_h):
            obj_id = row["object_id"]
            if not elem.Gene.has_key(obj_id, version):
                continue
            gene = elem.Gene[obj_id, version]
            synonym = row["ext_reference_id"]
            if synonym is not None:
                if bpattern.match(synonym):
                    if gene.bnumber or "obsolete" in synonym:
                        gene.synonyms.add(synonym)
                    else:
                        gene.bnumber = synonym
                else:
                    gene.synonyms.add(synonym)

def read_products(filename, version="default", sep="\t", comment="#",
        encoding=None, mode="rb", **kw_args):
    products = list()
    kw_args["mode"] = mode
    kw_args["encoding"] = encoding
    with open_file(filename, **kw_args) as (file_h, ext):
        iter_rowset = FILE_PARSERS.get(ext, iter_rowset_flat_file)
        for row in iter_rowset(file_h):
            # typo in RegulonDB files for molecular weight
            mw = row["molecular_weigth"]
            if not mw:
                mw = None
            ip = row["isoelectric_point"]
            if not ip:
                ip = None
            product = elem.Product(unique_id=row["product_id"],
                name_space=version,
                name=row["product_name"],
                molecular_weight=mw,
                isoelectric_point=ip)
            products.append(product)
    return products

def update_synonyms(filename, cls, version="default", sep="\t", comment="#",
        encoding=None, mode="rb", **kw_args):
    kw_args["mode"] = mode
    kw_args["encoding"] = encoding
    with open_file(filename, **kw_args) as (file_h, ext):
        iter_rowset = FILE_PARSERS.get(ext, iter_rowset_flat_file)
        for row in iter_rowset(file_h):
            obj_id = row["object_id"]
            if not cls.has_key(obj_id, version):
                continue
            inst = cls[obj_id, version]
            synonym = row["object_synonym_name"]
            if synonym:
                inst.synonyms.add(synonym)

def update_external(filename, cls, version="default", sep="\t", comment="#",
        encoding=None, mode="rb", **kw_args):
    kw_args["mode"] = mode
    kw_args["encoding"] = encoding
    with open_file(filename, **kw_args) as (file_h, ext):
        iter_rowset = FILE_PARSERS.get(ext, iter_rowset_flat_file)
        for row in iter_rowset(file_h):
            obj_id = row["object_id"]
            if not cls.has_key(obj_id, version):
                continue
            inst = cls[obj_id, version]
            synonym = row["ext_reference_id"]
            if synonym:
                inst.synonyms.add(synonym)

def link_gene_product(filename, version="default", sep="\t", comment="#",
        encoding=None, mode="rb", **kw_args):
    kw_args["mode"] = mode
    kw_args["encoding"] = encoding
    with open_file(filename, **kw_args) as (file_h, ext):
        iter_rowset = FILE_PARSERS.get(ext, iter_rowset_flat_file)
        for row in iter_rowset(file_h):
            gene_id = row["gene_id"]
            try:
                gene = elem.Gene[gene_id, version]
            except KeyError:
                LOGGER.warn("unknown gene: {0}".format(gene_id))
                continue
            prod_id = row["product_id"]
            # RegulonDB version 6.4 contains duplicate entries
            if gene.product:
                if gene.product.unique_id != prod_id:
                    LOGGER.warn("gene '{0}' and product '{1}' already exist in"\
                            " different context".format(gene_id, prod_id))
                continue
            try:
                gene.product = elem.Product[prod_id, version]
            except KeyError:
                LOGGER.warn("unknown product: {0}".format(prod_id))
                continue
            if gene.product.coded_from:
                if gene.product.coded_from.unique_id != gene_id:
                    LOGGER.warn("product '{0}' already has coding gene '{1}'".format(
                            gene.product.unique_id,
                            gene.product.coded_from.unique_id))
                continue
            gene.product.coded_from = gene

def read_gene_regulation(filename, version="default", sep="\t", comment="#",
        encoding=None, mode="rb", **kw_args):
    grn = GRN(name="Gene Regulatory Network")
    kw_args["mode"] = mode
    kw_args["encoding"] = encoding
    with open_file(filename, **kw_args) as (file_h, ext):
        iter_rowset = FILE_PARSERS.get(ext, iter_rowset_flat_file)
        for row in iter_rowset(file_h):
            try:
                eck12 = row["gene_id_regulator"]
                gene_u = elem.Gene[eck12, version]
            except KeyError:
                LOGGER.warn("unknown gene: {0}".format(eck12))
                continue
            try:
                eck12 = row["gene_id_regulated"]
                gene_v = elem.Gene[eck12, version]
            except KeyError:
                LOGGER.warn("unknown gene: {0}".format(eck12))
                continue
            interaction = row["generegulation_function"]
            grn.add_edge(gene_u, gene_v, key=FUNCTIONS[interaction])
    return grn

def read_sigma_factors(filename, version="default", sep="\t", comment="#",
        encoding=None, mode="rb", **kw_args):
    s_factors = list()
    kw_args["mode"] = mode
    kw_args["encoding"] = encoding
    with open_file(filename, **kw_args) as (file_h, ext):
        iter_rowset = FILE_PARSERS.get(ext, iter_rowset_flat_file)
        for row in iter_rowset(file_h):
            sigma_factor = elem.SigmaFactor(unique_id=row["sigma_id"],
                    name_space=version,
                    name=row["sigma_name"])
            sigma_factor.synonyms.add(row["sigma_synonyms"])
            try:
                eck12 = row["sigma_gene_id"]
                gene = elem.Gene[eck12, version]
            except KeyError:
                LOGGER.warn("unknown gene: {0}".format(eck12))
                s_factors.append(sigma_factor)
                continue
            sigma_factor.made_from.add(gene.product)
            gene.regulatory_product = sigma_factor
            sigma_factor.coded_from.add(gene)
            s_factors.append(sigma_factor)
    return s_factors

def read_transcription_factors(filename, version="default", sep="\t", comment="#",
        encoding=None, mode="rb", **kw_args):
    """
    TODO: transcription_factor.xml changes rapidly across versions, so depending
    on version we need to update various bits of information on the class
    instances.
    """
    kw_args["mode"] = mode
    kw_args["encoding"] = encoding
    t_factors = list()
    with open_file(filename, **kw_args) as (file_h, ext):
        iter_rowset = FILE_PARSERS.get(ext, iter_rowset_flat_file)
        for row in iter_rowset(file_h):
            t_factor = elem.TranscriptionFactor(
                unique_id=row["transcription_factor_id"],
                name_space=version,
                name=row["transcription_factor_name"])
            t_factors.append(t_factor)
    return t_factors

def update_product_transcription_factor_link(filename, version="default",
        sep="\t", comment="#", encoding=None, mode="rb", **kw_args):
    kw_args["mode"] = mode
    kw_args["encoding"] = encoding
    with open_file(filename, **kw_args) as (file_h, ext):
        iter_rowset = FILE_PARSERS.get(ext, iter_rowset_flat_file)
        for row in iter_rowset(file_h):
            prod_id = row["product_id"]
            try:
                product = elem.Product[prod_id, version]
            except KeyError:
                LOGGER.warn("unknown product: {0}".format(prod_id))
                continue
            tf_id = row["transcription_factor_id"]
            try:
                t_factor = elem.TranscriptionFactor[tf_id, version]
            except KeyError:
                LOGGER.warn("unknown transcription factor: {0}".format(prod_id))
                continue
            t_factor.made_from.add(product)
            t_factor.coded_from.add(product.coded_from)
            product.coded_from.regulatory_product = t_factor

def read_regulatory_interactions(filename, sep="\t", comment="#", encoding=None,
        mode="rb", **kw_args):
    """
    Extract regulatory interactions from relationships in RegulonDB files. Each
    interaction is given by a conformation ID and a promoter ID and the type of
    interaction.
    """
    kw_args["mode"] = mode
    kw_args["encoding"] = encoding
    interactions = list()
    with open_file(filename, **kw_args) as (file_h, ext):
        iter_rowset = FILE_PARSERS.get(ext, iter_rowset_flat_file)
        for row in iter_rowset(file_h):
            u = row["conformation_id"]
            v = row["promoter_id"]
            k = FUNCTIONS[row["ri_function"]]
            interactions.append((u, v, k))
    return interactions

def read_transcription_units(filename, version="default", sep="\t", comment="#",
        encoding=None, mode="rb", **kw_args):
    """
    Extract pairs of transcription unit and promoter IDs.

    Returns
    -------
    A map between promoter IDs and the various transcription units they occur
    in.
    """
    kw_args["mode"] = mode
    kw_args["encoding"] = encoding
    units = list()
    with open_file(filename, **kw_args) as (file_h, ext):
        iter_rowset = FILE_PARSERS.get(ext, iter_rowset_flat_file)
        for row in iter_rowset(file_h):
            tu_id = row["transcription_unit_id"]
            prom_id = row["promoter_id"]
            op_id = row["operon_id"]
            # RegulonDB 7.2 has some promoter ECK12 IDs that are not in
            # promoter.xml
            if prom_id:
                try:
                    prom = elem.Promoter[prom_id, version]
                except KeyError:
                    prom = None
                    LOGGER.warn("unknown promoter '%s' in transcription unit '%s'",
                            prom_id, tu_id)
            else:
                prom = None
            if op_id:
                try:
                    op = elem.Operon[op_id, version]
                except KeyError:
                    op = None
                    LOGGER.warn("unknown operon '%s' in transcription unit '%s'",
                            op_id, tu_id)
            else:
                op = None
            tu = elem.TranscriptionUnit(unique_id=tu_id,
                    name_space=version,
                    name=row["transcription_unit_name"],
                    promoter=prom, operon=op)
            units.append(tu)
    return units

def link_tu_genes(filename, version="default", sep="\t", comment="#",
        encoding=None, mode="rb", **kw_args):
    """
    Link transcription units (TU) with their corresponding genes.

    Parameters
    ----------
    filename: str
        Relative or absolute path to file that contains the  RegulonDB information.

    Returns
    -------
    A dict with keys that are TU IDs and values that are gene IDs contained
    within that TU.
    """
    kw_args["mode"] = mode
    kw_args["encoding"] = encoding
    t_units = list()
    with open_file(filename, **kw_args) as (file_h, ext):
        iter_rowset = FILE_PARSERS.get(ext, iter_rowset_flat_file)
        for row in iter_rowset(file_h):
            tu_id = row["transcription_unit_id"]
            try:
                t_unit = elem.TranscriptionUnit[tu_id, version]
            except KeyError:
                LOGGER.error("unknown transcription unit '%s', please parse"\
                        " those first.", tu_id)
                continue
            gene_id = row["gene_id"]
            try:
                gene = elem.Gene[gene_id, version]
            except KeyError:
                LOGGER.error("unknown gene '%s', please parse those first.", gene_id)
                continue
            t_unit.genes.append(gene)
            gene.transcription_units.add(t_unit)
            t_units.append(t_unit)
    begin = attrgetter("position")
    for tu in t_units:
        if all(gene.strand == "reverse" for gene in tu.genes):
            tu.genes.sort(key=begin, reverse=True)
        elif all(gene.strand == "forward" for gene in tu.genes):
            tu.genes.sort(key=begin)
        else:
            LOGGER.error("conflicting strand information in transcription unit"\
                    " '%s'", tu.unique_id)

def read_promoters(filename, version="default", sep="\t", comment="#",
        encoding=None, mode="rb", **kw_args):
    """
    Extract promoter information.

    Parameters
    ----------
    filename: str
        Relative or absolute path to file that contains the  RegulonDB information.

    Returns
    -------
    """
    kw_args["mode"] = mode
    kw_args["encoding"] = encoding
    promoters = list()
    with open_file(filename, **kw_args) as (file_h, ext):
        iter_rowset = FILE_PARSERS.get(ext, iter_rowset_flat_file)
        for row in iter_rowset(file_h):
            strand = row["promoter_strand"]
            if not strand:
                strand = None
            prom = elem.Promoter(unique_id=row["promoter_id"],
                    name_space=version,
                    name=row["promoter_name"],
                    strand=strand,
                    pos_1=misc.convert2numeral(row["pos_1"]),
                    sequence=row["promoter_sequence"],
                    sigma_factor=row["sigma_factor"].split(","),
                    note=row["promoter_note"]
            )
            promoters.append(prom)
    return promoters

def read_operons(filename, version="default", sep="\t", comment="#",
        encoding=None, mode="rb", **kw_args):
    """
    Extract operon information.

    Parameters
    ----------
    filename: str
        Relative or absolute path to file that contains the  RegulonDB information.

    Returns
    -------
    """
    kw_args["mode"] = mode
    kw_args["encoding"] = encoding
    operons = list()
    with open_file(filename, **kw_args) as (file_h, ext):
        iter_rowset = FILE_PARSERS.get(ext, iter_rowset_flat_file)
        for row in iter_rowset(file_h):
            gene_left = misc.convert2numeral(row["firstgeneposleft"], int)
            if not gene_left:
                gene_left = None
            gene_right = misc.convert2numeral(row["lastgeneposright"], int)
            if not gene_right:
                gene_right = None
            reg_left = misc.convert2numeral(row["regulationposleft"], int)
            if not reg_left:
                reg_left = None
            reg_right = misc.convert2numeral(row["regulationposright"], int)
            if not reg_right:
                reg_right = None
            strand = row["operon_strand"]
            if not strand:
                strand = None
            op = elem.Operon(
                    unique_id=row["operon_id"],
                    name_space=version,
                    name=row["operon_name"],
                    strand=strand,
                    gene_position_start=gene_left,
                    gene_position_end=gene_right,
                    regulation_position_start=reg_left,
                    regulation_position_end=reg_right
            )
            operons.append(op)
    return operons

def update_operons(operons, promoters, genes, **kw_args):
    """
    Extract operon information.

    Parameters
    ----------

    Returns
    -------
    """
    promoters = list(prom for prom in promoters if prom.pos_1 is not None)
    prom_i = np.arange(len(promoters), dtype=int)
    prom_pos = np.array([prom.pos_1 for prom in promoters])
    genes = list(gene for gene in genes if gene.position is not None)
    gene_i = np.arange(len(genes), dtype=int)
    gene_start = np.array([gene.position_start for gene in genes])
    gene_end = np.array([gene.position_end for gene in genes])
    begin = attrgetter("position")
    for op in operons:
        mask = np.logical_and((prom_pos >= op.regulation_position_start),
                (prom_pos <= op.regulation_position_end))
        for prom in (promoters[i] for i in prom_i[mask]):
            if prom.strand == op.strand:
                op.promoters.add(prom)
        mask = np.logical_and((gene_start >= op.gene_position_start),
                (gene_end <= op.gene_position_end))
        for gene in (genes[i] for i in gene_i[mask]):
            if gene.strand == op.strand:
                op.genes.append(gene)
                gene.operons.add(op)
        if op.strand == "reverse":
            op.genes.sort(key=begin, reverse=True)
        else:
            op.genes.sort(key=begin)

def read_conformations(filename, version="default", sep="\t", comment="#",
        encoding=None, mode="rb", **kw_args):
    """
    Extract conformation information.

    Parameters
    ----------
    filename: str
        Relative or absolute path to file that contains the  RegulonDB information.

    Returns
    -------
    """
    kw_args["mode"] = mode
    kw_args["encoding"] = encoding
    conformations = list()
    with open_file(filename, **kw_args) as (file_h, ext):
        iter_rowset = FILE_PARSERS.get(ext, iter_rowset_flat_file)
        for row in iter_rowset(file_h):
            tf_id = row["transcription_factor_id"]
            try:
                t_factor = elem.TranscriptionFactor[tf_id, version]
            except KeyError:
                LOGGER.warn("unknown transcription factor %s", tf_id)
                LOGGER.warn("Please parse transcription factor information before"\
                        " parsing conformations.")
                continue
            conf = elem.Conformation(
                    unique_id=row["conformation_id"],
                    name_space=version,
                    tf=t_factor,
                    state=row["final_state"],
                    interaction=row["interaction_type"],
                    conformation_type=row.get("conformation_type", None), # version dependent
                    apo_holo=row.get("apo_holo_conformation", None) # version dependent
            )
            t_factor.conformations.add(conf)
            conformations.append(conf)
    return conformations

def read_tf_gene_network(filename, name2tf, name2gene, sep="\t",
        encoding=None, mode="rb", comment="#", **kw_args):
    """
    Retrieve the TRN from a RegulonDB flat file.

    Parameters
    ----------
    filename: str
        The path to the file containing the interaction information.
    name2tf: dict
        The transcription factors in the parsed interactions are given by
        their name. A map between names and objects is required.
    name2gene: dict
        The genes in the parsed interactions are given by their name. A map
        between names and objects is required.
    sep: str (optional)
        The column separator; not likely to change.
    encoding: str (optional)
        The file encoding.
    mode: str (optional)
        The mode used to open a file, should always be read binary.
    """
    trn = TRN(name=filename)
    kw_args["encoding"] = encoding
    kw_args["mode"] = mode
    with open_file(filename, **kw_args) as (file_h, ext):
        iter_rowset = FILE_PARSERS.get(ext, iter_rowset_flat_file)
        for row in iter_rowset(file_h, header=["tf", "gene", "effect",
                "evidence"]):
            first = name2tf[row["tf"]]
            second = name2gene[row["gene"]]
            trn.add_edge(first, second, key=FUNCTIONS[row["effect"]],
                    evidence=row["evidence"])
    return trn

#def read_sigma_gene_network(genes, filename, sep="\t", comment="#",
#        encoding=None, mode="rb", **kw_args):
#
#    links = list()
#    # read information from the file
#    kw_args["mode"] = mode
#    kw_args["encoding"] = encoding
#    with open_file(filename, **kw_args) as (file_h, ext):
#        if ext != ".txt":
#            raise NotImplementedError
#        for row in read_tabular(file_h, sep=sep, comment=comment):
#            if row[2] == "-":
#                links.append((row[0], row[1], -1, row[3]))
#            elif row[2] == "+":
#                links.append((row[0], row[1], 1, row[3]))
#            elif row[2] == "?":
#                links.append((row[0], row[1], 0, row[3]))
#            elif row[2] == "+-":
#                links.append((row[0], row[1], 2, row[3]))
#            else:
#                LOGGER.warn(row)
#    # map sigma factor names to objects
#    sigma_factors = [gene.regulatory_product for gene in genes\
#            if type(gene.regulatory_product) == elem.SigmaFactor]
#    sigma_names = set([quad[0] for quad in links])
#    name2sigma = dict()
#    for name in sigma_names:
#        name2sigma[name] = find_element(name, sigma_factors)
#    for (name, tar) in name2sigma.items():
#        if tar is None:
#            LOGGER.warn("sigma factor '{0}' not found".format(name))
#    # map gene names to objects
#    gene_names = set([quad[1] for quad in links])
#    name2gene = dict()
#    for name in gene_names:
#        name2gene[name] = find_element(name, genes)
#    for (name, tar) in name2gene.items():
#        if tar is None:
#            LOGGER.warn("gene '{0}' not found".format(name))
#    # now apply the maps to the interactions
#    interactions = [(name2sigma[quad[0]], name2gene[quad[1]], quad[2])\
#            for quad in links]
#    return interactions

#def parse_regulondb_entry(eck12, dscrptn):
#    dscrptn = dscrptn.split("\n")
#    gene = False
#    product = False
#    info = dict()
#    prod_info = dict()
#    blattner = re.compile(r"b\d+")
#    location = re.compile(r"\d+")
#    phantom = 0
#    for line in dscrptn:
#        if line.startswith("GENE"):
#            gene = True
#        elif line.startswith("PRODUCT"):
#            gene = False
#            product = True
#        elif gene and line.startswith("\t"):
#            line = line.strip()
#            tmp = line.split(":")
#            if tmp[0].startswith("Name"):
#                name = tmp[1].strip()
#                if name == "Phantom Gene":
#                    phantom += 1
#                    info["name"] = "%s %d" % (name, phantom)
#                else:
#                    info["name"] = name
#            elif tmp[0].startswith("Synonym"):
#                info["synonyms"] = list()
#                for name in tmp[1].split(","):
#                    name = name.strip()
#                    if blattner.match(name):
#                        info["bnumber"] = name
#                    else:
#                        info["synonyms"].append(name)
#            elif tmp[0].startswith("Map"):
#                info["position"] = location.findall(tmp[1])
#                info["position"] = map(int, info["position"])
#                info["position"] = tuple(info["position"])
#            elif tmp[0].startswith("Strand"):
#                info["strand"] = tmp[1].strip()
#        elif product and line.startswith("\t"):
#            line = line.strip()
#            tmp = line.split(":")
#            if tmp[0].startswith("Name"):
#                prod_info["name"] = tmp[1].strip()
#            elif tmp[0].startswith("Synonym"):
#                prod_info["synonyms"] = list()
#                for name in tmp[1].split(","):
#                    name = name.strip()
#                    prod_info["synonyms"].append(name)
#            elif tmp[0].startswith("GO"):
#                prod_info["go"] = int(tmp[2])
#        else:
#            gene = False
#            product = False
#            break
#    tf = TranscriptionFactor(**prod_info)
#    return Gene(eck12=eck12, product=tf, **info)

def lookup_gene_information(eck12_nums, num_threads=20,
        wsdl="http://regulondb.ccg.unam.mx/webservices/Gene.jws?wsdl"):
    """
    A threaded method that extracts gene information from RegulonDB.

    Parameters
    ----------
    eck12_nums: list
        A collection of ECK12 identifiers used by RegulonDB whose information
        should be retrieved.
    wsdl: str (optional)
        URL of the RegulonDB WSDL server.
    num_threads: int
        The number of desired simultaneous connections to the WSDL server.

    Notes
    -----
    Requires SOAPpy and an active internet connection.
    """

    from .wsdl import ThreadedWSDLFetcher
    from Queue import Queue
    # use a threaded approach to server querying
    tasks = Queue()
    for i in range(num_threads):
        thrd = ThreadedWSDLFetcher(tasks, wsdl)
        thrd.start()
    # get all gene descriptions from RegulonDB
    descriptions = list()
    for gene in eck12_nums:
        tasks.put(("getGene", gene, descriptions))
    tasks.join()
    return descriptions

def draw_relationships(file_contents, emph=list(), ignore=["key_id_org"],
        title="", font_size=14.0, width=16.54, height=11.69):
    """
    Draw a graph with relationships between tags in RegulonDB XML files.

    Parameters
    ----------
    file_contents: dict
        A dictionary where keys are filenames and values are lists of tag names
        of the corresponding file.
    emph: list
        A list of tags that should be emphasized by colour.
    ignore: list
        A list of tags to ignore when drawing relationships.
    title: str
        A title description for the printed graph.

    Notes
    -----
    Requires pygraphviz.
    """
    if len(emph) > len(misc.BREWER_SET1):
        raise PyOrganismError("number of objects to be emphasized ({0:d}) is"\
                " greater than the number of colours available ({1:d})",
                len(emph), len(misc.BREWER_SET1))
    pgv = misc.load_module("pygraphviz")
    colour_choice = dict(zip(emph, misc.BREWER_SET1))
    graph = pgv.AGraph(name="RegulonDB File-Relationships", strict=True,
            directed=False, rankdir="TB")
    graph.graph_attr["labelloc"] = "t"
    graph.graph_attr["label"] = title
    graph.graph_attr["fontsize"] = font_size * 1.5
    graph.graph_attr["ranksep"] = "0.1 equally"
    graph.graph_attr["size"] = (width, height)
    graph.graph_attr["ratio"] = "compress"
    graph.node_attr["shape"] = "none"
    graph.node_attr["fontsize"] = font_size
    for (name, attrs) in file_contents.items():
        label = ["<<TABLE BORDER=\"0\" CELLBORDER=\"1\" CELLSPACING=\"0\""\
                " CELLPADDING=\"4\">"]
        label.append("<TR><TD BGCOLOR=\"#A4A4A4\"><B>{0}</B></TD></TR>".format(name))
        for (i, attr) in enumerate(attrs):
            if attr in emph:
                label.append("<TR><TD PORT=\"f{0:d}\" BGCOLOR=\"{1}\">{2}</TD></TR>".format(i,
                            colour_choice[attr], attr))
            else:
                label.append("<TR><TD PORT=\"f{0:d}\">{1}</TD></TR>".format(i,
                    attr))
        label.append("</TABLE>>")
        graph.add_node(name, label="\n".join(label))
    nodes = file_contents.keys()
    for i in range(len(nodes) - 1):
        node_u = nodes[i]
        attr_u = file_contents[node_u]
        for j in range(i + 1, len(nodes)):
            node_v = nodes[j]
            attr_v = file_contents[node_v]
            shared = set(attr_u).intersection(set(attr_v))
            for attr in shared:
                if attr in ignore:
                    continue
                u = attr_u.index(attr)
                v = attr_v.index(attr)
                if attr in emph:
                    graph.add_edge(node_u, node_v,
                            tailport="f{0:d}".format(u), headport="f{0:d}".format(v),
                            color=colour_choice[attr])
                else:
                    graph.add_edge(node_u, node_v,
                            tailport="f{0:d}".format(u), headport="f{0:d}".format(v))
    sub_attr = dict()
    nodes = graph.nodes()
    nodes.sort(key=lambda n: graph.degree(n))
    maxi = nodes[-1: -len(nodes) / 4]
    nodes = nodes[:-len(nodes) / 4]
    zeros = [node for (node, deg) in graph.degree_iter() if deg == 0]
    for n in zeros:
        nodes.remove(n)
    graph.add_subgraph(maxi, name="input", rank="source", **sub_attr)
    graph.add_subgraph(nodes, name="middle", **sub_attr)
    graph.add_subgraph(zeros, rank="sink", **sub_attr)
    return graph

max_stars_repo_name: Midnighter/pyorganism

max_issues_repo_name: Midnighter/pyorganism

max_forks_repo_name: Midnighter/pyorganism


text:
from pathlib import Path

import cv2
import numpy as np
import torch

from custom_models.model import GrayscaleModel, GaussianBlur

def export():
    output_dir = Path(__file__).parent / 'out'
    output_dir.mkdir(parents=True, exist_ok=True)
    export_onnx(output_dir=output_dir)
    print('Done.')

def export_onnx(output_dir):
    """
    Exports the model to an ONNX file.
    """
    # Channels Last
    shape = (1, 300, 300, 3)
    model = GaussianBlur(shape=shape, dtype=torch.float)
    X = torch.ones(shape, dtype=torch.uint8)
    torch.onnx.export(
        model,
        X,
        f'{output_dir.as_posix()}/model.onnx',
        opset_version=9,
        do_constant_folding=True
    )

def preview():
    capture = cv2.VideoCapture(0)
    model = None
    while True:
        _, color = capture.read()
        if not model:
            shape = (1, ) + color.shape
            model = GaussianBlur(shape=shape, dtype=torch.uint8).eval()

        # Numpy -> torch.Tensor
        cv2.imshow('Original', color)
        color = torch.from_numpy(color)
        color = torch.unsqueeze(color, dim=0)
        out = model(color)[0].detach().numpy()
        # Channels Last
        out = out.astype(np.uint8)
        cv2.imshow('Gray', out)
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break

    capture.release()
    cv2.destroyAllWindows()

if __name__ == "__main__":
    export()

max_stars_repo_name: tikurahul/depthai

max_issues_repo_name: tikurahul/depthai

max_forks_repo_name: tikurahul/depthai


text:
# Copyright (c) OpenMMLab. All rights reserved.
import numpy as np
import torch
import torch.nn.functional as F

from mmcls.models.utils.augment.builder import AUGMENT
from .cutmix import BatchCutMixLayer
from .utils import one_hot_encoding

@AUGMENT.register_module(name='BatchResizeMix')
class BatchResizeMixLayer(BatchCutMixLayer):
    r"""ResizeMix Random Paste layer for a batch of data.

    The ResizeMix will resize an image to a small patch and paste it on another
    image. It's proposed in `ResizeMix: Mixing Data with Preserved Object
    Information and True Labels <https://arxiv.org/abs/2012.11101>`_

    Args:
        alpha (float): Parameters for Beta distribution to generate the
            mixing ratio. It should be a positive number. More details
            can be found in :class:`BatchMixupLayer`.
        num_classes (int): The number of classes.
        lam_min(float): The minimum value of lam. Defaults to 0.1.
        lam_max(float): The maximum value of lam. Defaults to 0.8.
        interpolation (str): algorithm used for upsampling:
            'nearest' | 'linear' | 'bilinear' | 'bicubic' | 'trilinear' |
            'area'. Default to 'bilinear'.
        prob (float): The probability to execute resizemix. It should be in
            range [0, 1]. Defaults to 1.0.
        cutmix_minmax (List[float], optional): The min/max area ratio of the
            patches. If not None, the bounding-box of patches is uniform
            sampled within this ratio range, and the ``alpha`` will be ignored.
            Otherwise, the bounding-box is generated according to the
            ``alpha``. Defaults to None.
        correct_lam (bool): Whether to apply lambda correction when cutmix bbox
            clipped by image borders. Defaults to True
        **kwargs: Any other parameters accpeted by :class:`BatchCutMixLayer`.

    Note:
        The :math:`\lambda` (``lam``) is the mixing ratio. It's a random
        variable which follows :math:`Beta(\alpha, \alpha)` and is mapped
        to the range [``lam_min``, ``lam_max``].

        .. math::
            \lambda = \frac{Beta(\alpha, \alpha)}
            {\lambda_{max} - \lambda_{min}} + \lambda_{min}

        And the resize ratio of source images is calculated by :math:`\lambda`:

        .. math::
            \text{ratio} = \sqrt{1-\lambda}
    """

    def __init__(self,
                 alpha,
                 num_classes,
                 lam_min: float = 0.1,
                 lam_max: float = 0.8,
                 interpolation='bilinear',
                 prob=1.0,
                 cutmix_minmax=None,
                 correct_lam=True,
                 **kwargs):
        super(BatchResizeMixLayer, self).__init__(
            alpha=alpha,
            num_classes=num_classes,
            prob=prob,
            cutmix_minmax=cutmix_minmax,
            correct_lam=correct_lam,
            **kwargs)
        self.lam_min = lam_min
        self.lam_max = lam_max
        self.interpolation = interpolation

    def cutmix(self, img, gt_label):
        one_hot_gt_label = one_hot_encoding(gt_label, self.num_classes)

        lam = np.random.beta(self.alpha, self.alpha)
        lam = lam * (self.lam_max - self.lam_min) + self.lam_min
        batch_size = img.size(0)
        index = torch.randperm(batch_size)

        (bby1, bby2, bbx1,
         bbx2), lam = self.cutmix_bbox_and_lam(img.shape, lam)

        img[:, :, bby1:bby2, bbx1:bbx2] = F.interpolate(
            img[index],
            size=(bby2 - bby1, bbx2 - bbx1),
            mode=self.interpolation)
        mixed_gt_label = lam * one_hot_gt_label + (
            1 - lam) * one_hot_gt_label[index, :]
        return img, mixed_gt_label

max_stars_repo_name: youqingxiaozhua/mmclassification

max_issues_repo_name: youqingxiaozhua/mmclassification

max_forks_repo_name: youqingxiaozhua/mmclassification


text:
# coding=utf-8
import numpy as np
from configs.hyperparams import TrainConfig

def list2array(_list):
    return np.array(list(map(float, _list)))

def array2string(array):
    return ' '.join(map(str, array.tolist()))

class EmbdMapper:
    def __init__(self, data_path, config):
        self.data_path = data_path
        # special char
        self.eos_char = config.eos_char
        self.go_char = config.go_char
        self.unk_char = config.unk_char
        self.pad_char = config.pad_char
        # load embedding
        self.num, self.dim, self.embd_dict = self.load_embd()

    def load_embd(self):
        embd_dict = dict()
        with open(self.data_path, "r") as f:
            # read meta data
            meta_line = f.readline()
            _, dim = meta_line.split()
            dim = int(dim)
            while 1:
                lines = f.readlines(1000)
                if not lines:
                    break
                for line in lines:
                    chars, *scalars = line[:-1].split()
                    if len(chars) > 1:
                        continue
                    # extend 3 dimension for embedding another 4 special chars.
                    scalars.extend([0, 0, 0])
                    array = list2array(scalars)
                    print(sum(array))
                    # print(np.sum(np.square(array)))
                    embd_dict[chars] = array
        # add 4 special chars
        zeros = np.zeros(shape=(1, dim))
        print(zeros.shape)
        embd_dict[self.go_char] = np.concatenate((zeros, [[1, 0, 0]]), axis=1)
        embd_dict[self.eos_char] = np.concatenate((zeros, [[0, 1, 0]]), axis=1)
        embd_dict[self.unk_char] = np.concatenate((zeros, [[0, 0, 1]]), axis=1)
        embd_dict[self.pad_char] = np.concatenate((zeros, [[0, 0, 0]]), axis=1)
        num = len(embd_dict.keys())
        print('number of vector:{}, dimension:{}'.format(num, dim+3))
        return num, dim+3, embd_dict

    def map(self, char):
        if char not in self.embd_dict.keys():
            raise KeyError
        return self.embd_dict[char]

    def get_char2idx(self):
        char2idx = {char: idx for idx, char in enumerate(self.embd_dict.keys())}
        return char2idx

    def get_lookup_table(self):
        return list2array(self.embd_dict.values())

    def save(self, path):
        embd_list = list()
        for k in self.embd_dict.keys():
            embd_list.append('{} {}\n'.format(k, array2string(self.embd_dict[k])))
        sorted(embd_list)
        with open(path, 'w', newline='\n') as f:
            f.writelines(['{} {}\n'.format(self.num, self.dim)])
            f.writelines(embd_list)

def main():
    path = 'data/embd/sgns.renmin.char'
    config = TrainConfig()
    em = EmbdMapper(path, config)
    em.save('data/embd/sgns.renmin.char.reduce')

if __name__ == '__main__':
    main()


text:
"""Implements fast arithmetic of bit vectors

A bit vector b = (b[0],...,b[n-1]) , b[i] in (0,1) is coded as the
integer sum( [ b[i] * 2**i  for i in range(n) ] ). The is a bit
inconvenient to  read for a human, but with this encoding functions
operating on bit vectors can be defined and implemented in a much more
consistent way. Function bin() also allows to print the bits of an
integer in reverse order.  The first index of a vector or matrix is
always zero, as usual in Python or C.

Generally speaking, bit vectors are mainly used as modules with a
group operating on them. Elements of such groups may be matrices
or permutations on bit vectors. Here bit vectors are considered
as row vectors, and bit matrices or permutation always operate
on the right. Thus  x * g1 * g2, where x is a bit vector and g1,
g2 are elements of a group, means that first g1 is applied to v
and then g2 is applied to the bit vector v * g1.

Bit matrices are represented as lists of integers, where the i-th
element of the list encodes the i-th row vector of the matrix.
Function bit_mat_mul() multiplies an arbitrary number of
matrices. The first factor may also be a bitvector. In the same
way as an integer in binary representation can be preceeded by
an arbitrary number of of zero bits (without changing its value),
we assume that a list representing a matrix can be followed by
an arbitrary number of zero entries so that multiplcation of
bit matrices is always possible.

A permutation p is also represented as a list of integers.
Here p[i] = k means that the bit at position i is mapped to
position k. More strictly, we will interpret a permutation p
represented by a list [ p[0], p[1],...,p][n-1] ] with
p[i] = k as a bit matrix b with b[i,j]  = 1 if j==k and
b[i,j] = 0 if j != k. Function  bit_perm_mul() multiplies a
bit vector or a permutation with an arbitrary number of
permutations. Function  bit_perm_2_mat() transforms a
permutation into an equivalent bit matrix.

"""
from __future__ import absolute_import, division, print_function
from __future__ import  unicode_literals

from random import randint
import types
from operator import __or__ , __xor__
from functools import reduce
from collections.abc import Iterable
from numbers import Integral

import numpy
from numpy import array, zeros, uint8, uint16, uint32, uint64, int64

def isinteger(x):
    """Deprecated, use numbers.Integral() instead!"""
    return  isinstance(x, (Integral, numpy.integer))

def lmap(*args):
    """For compatibility with map() in python 2"""
    return list(map(*args))

def lrange(*args):
    """For compatibility with range() in python 2"""
    return list(range(*args))

def bitlen(x):
    """returns the length of the binary representation of an integer x >= 0"""
    assert x >= 0, "bitlen(x) requires nonegative integer x"
    try:
        return (int(x)).bit_length()
    except:  # for python versions < 2.7 or 3.1:
        l, m, sh = 0, 0x10000, 16
        while x >= m:
            #x, l = x>>sh, l+sh
            m, sh = m << sh, sh << 1
        #assert m == 1 << sh
        while sh:
            if x >= m: x, l = x>>sh, l+sh
            sh >>= 1
            m = m >> sh
        while x: x, l = x>>1, l+1
        return l

def hibit(x):
   """return the position of the highest bit of an integer"""
   if x==0: raise ZeroDivisionError("Zero has no highest bit")
   return bitlen(x) - 1

def v2(*args):
   """Return the 2-adic value of an integer or of a set of integers.

   For an integer ``i = o * 2**k``, ``o`` odd, we have ``v2(i) = k``.

   If ``x`` is iterable or a numpy array then ``v2(x)`` is the
   minimum of the values ``v2(i)`` for all entries of ``i`` of ``x``.

   If several arguments ``x_1,...,x_n`` are given then
   ``v2(x_1,...,x_n)`` is the minimum of all values ``v2(x_i)``.

   The function raises ZeroDivisionError if all integers occuring as
   arguments (or contained in an argument) are zero.
   """
   x = 0
   for a in args:
       if isinstance(a, Integral):
           x |= int(a)
       elif isinstance(a, numpy.ndarray):
           try:
               x |= int(reduce(__or__, a.reshape(-1), 0))
           except TypeError:
               if issubclass(a.dtype.type, np.integer):
                    # workaround for the case a.dtype == numpy.uint64
                    x |= reduce(__or__, map(int,a.reshape(-1)), 0)
               else:
                   raise
       elif isinstance(a, Iterable):
           x |= int(reduce(__or__, a, 0))
       else:
          err = "Function v2() undefined for a %s object"
          raise TypeError(err % type(a))
   l = (x & -x).bit_length() - 1
   if l >= 0:
       return l
   raise ZeroDivisionError("Function v2(x) is undefined for x = 0")

def bits2list(x):
   """returns the list of positions of one-bits of integer x

   List of bit positions is sorted in ascending order
   """
   l = bitlen(x)
   return [ i for i in range(l) if (x >> i) & 1 ]

def list2bits(lst):
   """return number x with bit i of x  set for all i in the list 'lst' """
   return reduce(__or__, [ 1<<i for i in lst ], 0)

def list_xor(ListList):
   """Given a list of lists, the list of  XOR sums of the cartesian product are returned"""
   if len(ListList) <= 1:
       return ListList[0] if len(ListList) else []
   d = len(ListList) >>1
   l1, l2 = ListXor(ListList[:d]), ListXor(ListList[d:])
   return [x ^ y for x in l1 for y in l2]

def iter_bitweight(k,n):
    """Iterate through all integers 0 <= x < n with bit weight k"""
    if k > 0:
        x = (1 << k) - 1  # smallest integer of bit weight k
        while x < n:
            yield x
            # use Gosper's hack to get next integer x of bit weight k
            c = x & -x
            r = x + c
            x = (((x ^ r) >> 2) // c) | r
    elif n > 0:
        yield 0

def iter_ascending_bitweight(n):
    """Iterate through all integers 0 <= x < n by ascending bit weight"""
    if n > 0:
        yield 0
        k = 1
        while 1 << k <= n:
            for y in iter_bitweight(k,n):
                yield y
            k = k + 1

def bw24(x):
    """Return the bit weight of the lowest 24 bits of x"""
    x = (x & 0x555555) + ((x & 0xaaaaaa) >> 1)
    x = (x & 0x333333) + ((x & 0xcccccc) >> 2)
    x = (x + (x >> 4)) & 0xf0f0f
    return (x + (x >> 8) + (x >> 16)) & 0x1f

def bitweight(x):
   """return the bit weight of the  word 'x'. """
   assert x >= 0, "bitweight(x) requires integer x >= 0"
   w = 0
   while x:
        w += bw24(x)
        x >>= 24
   return w

def bitparity(x):
   """return the bit parity of the  word 'x'. """
   assert x >= 0, "bitparity(x) requires integer x >= 0"
   while x > 0xffffffff:
       x = (x >> 32) ^ (x & 0xffffffff)
   x ^= x >> 16
   x ^= x >> 8
   x ^= x >> 4
   return (0x6996 >> (x & 15)) & 1

def reverse24(x, k=24):
    """reverses order of bits 0,...,k-1 of an integer. Default is k=24.

    If any bits apart form bit 0,...,k-1 are set, an execption is raised.
    """
    if x & -(1 << k):
        raise ValueError("Too high bits are set for bit reversal")
    return sum( ((x >> (k-i-1)) & 1) << i  for  i in range(k) )

def bin(x, l=24, d=8, reverse=False):
   """returns the binary representation of an 'l'-bit integer 'x' as a string

   Leading zeros are printed, blanks are inserted between 'd'-bit blocks.
   'd' may also be a list of bit positions where to insert blanks. If
   'reverse' is True, the string starts with bit 0 (which is fine for bit
   vectors), otherwise it starts with bit l-1 (as usual for integers).
   """
   s = ""
   if not isinstance(d, list):  d=range(d,l,d)
   if reverse:
      for i in range(l):
         if x & (1<<i): s += '1'
         else: s += '0'
         if i+1 in d: s += ' '
   else:
      for i in range(l-1,-1,-1):
         if x & (1<<i): s += '1'
         else: s += '0'
         if i in d: s += ' '
   return s

def binbv(x, l=24, d=8):
   """shorthand for bin(x, l, d, reverse=True)"""
   return bin(x, l, d, reverse=True)

def pivot_binary_high(basis):
   """Pivot over a basis of integers considered as bit vectors

   Here 'basis' list a list of integers corresponding to bit vectors,
   and the elements of the list 'basis' are considered as the rows of a
   bit matrix defining a basis of a linear space in a vector space
   over {0,1}.

   The function performs Gaussian elimination to change the basis to
   a standard form. Here pivoting over bit columns is done, so that
   at the end each pivoted bit column contains exactly one nonzero bit.
   Redundant rows of the original basis are removed.

   Bit columns are not exchanged. The function returns a pair
   (basis, columns), where 'basis' is the modified basis and 'columns'
   is a list of integers containing the index of the pivoted column
   used in the corresponding row of the basis. Here index i corresponds
   to the bit column with valence i.

   The highest possible bit column is always taken for pivoting.
   """
   basis, columns = list(basis)[:], []
   for i in range(len(basis)):
       m = max(basis[i:])
       j = basis[i:].index(m) + i
       try:
           mi = hibit(m)
       except:
           del basis[i:]
           break
       else:
           mask = 1 << mi
           columns.append(mi)
           basis[i], basis[j] = m, basis[i]
           for j in range(0, len(basis)):
               if j != i and basis[j] & mask:
                    basis[j] ^= m
   return basis, columns

def pivot_binary_low(basis):
   """Pivot over a basis of integers considered as bit vectors

   Here 'basis' list a list of integers corresponding to bit vectors,
   and the elements of the list 'basis' are considered as the rows of a
   bit matrix defining a basis of a linear space in a vector space
   over {0,1}.

   The function performs Gaussian elimination to change the basis to
   a standard form. Here pivoting over bit columns is done, so that
   at the end each pivoted bit column contains exactly one nonzero bit.
   Redundant rows of the original basis are removed.

   Bit columns are not exchanged. The function returns a pair
   (basis, columns), where 'basis' is the modified basis and 'columns'
   is a list of integers containing the index of the pivoted column
   used in the corresponding row of the basis. Here index i corresponds
   to the bit column with valence i.

   The lowest possible bit column is always taken for pivoting.
   """
   basis, columns = basis[:], []
   for i in range(len(basis)):
       m = reduce(__or__, basis[i:], 0)
       try:
           mi = v2(m)
       except:
           del basis[i:]
           break
       else:
           mask = 1 << mi
           j = [x & mask for x in basis[i:]].index(mask) + i
           columns.append(mi)
           m = basis[j]
           basis[i], basis[j] = m, basis[i]
           for j in range(0, len(basis)):
               if j != i and basis[j] & mask:
                   basis[j] ^= m
   return basis, columns

_maxbits = {uint8:8, uint16:16, uint32:32}

def lin_table(lst, dtype = uint32, t0 = 0):
    """return a linear table t with t[0]=t0, t[2**i]=lst(i).

    For all entries i,j of t we have t[i^j] ^ t[0] = t[i] ^ t[i].
    t[0] is set to the value t0 (default is 0).

    The table ist returned as a numpy array of type 'dtype',
    with dtype = uint8, unit16 or uint 32.
    """
    if not 0 <= (reduce(__or__,lst,0) | t0) <= 1 << _maxbits[dtype]:
        print( hex(reduce(__or__,lst,0) | t0) )
        raise ValueError("entries too large for table")
    a = numpy.zeros(1 << len(lst), dtype = dtype)
    a[0] = t0
    for i, x in enumerate(lst):
        a[1<<i:2<<i].fill(x)
        a[1<<i:2<<i] ^= a[0:1<<i]
    return a

def bit_mat_iszero(a):
    """return 0 if a is a zero matrix"""
    if isinteger(a):
         return a == 0
    return sum(lmap(abs,a)) == 0

def bit_mat_mul(a, *args):
    """ Multiplies 'a' with an arbitrary number of bit matrices.

    Parameter 'a' may be a bit vector or a list representing a bit matrix.
    Representation of bit vectors and matrices see comment on module.
    The result returned has the same shape as parameter 'a'.
    """
    if len(args) == 1:
        b = args[0]
        if isinteger(a):
            return reduce(
                   __xor__, [x for i,x in enumerate(b) if a & (1<<i)], 0 )
        else:
            return [ reduce(
                  __xor__, [x for i,x in enumerate(b) if y & (1<<i)], 0 )
               for y in a ]
    if len(args) == 0:
        return a
    return  bit_mat_mul(bit_mat_mul(a, args[0]), *args[1:])

def bit_mat_transpose(a, ncolumns=None):
    """return the  transpose of bit Matrix a.

    Optionally, the number of columns of a may be given.

    """
    ncols = bitlen(reduce(__or__, a, 0))
    if ncolumns is None:
        ncolumns = ncols
    elif ncols > ncolumns:
         raise ValueError( "Bad entry in bit matrix" )
    t = [0] * ncolumns
    for i, l in enumerate(a):
        for j in range(ncolumns):
            if (l >> j) & 1:
                 t[j] ^= 1 << i
    return t

def bit_mat_inverse(a):
    """return the inverse of bit Matrix a."""
    msg = "Bit matrix is not invertible"
    ncols = bitlen(reduce(__or__, a, 0))
    if ncols != len(a):
         raise ZeroDivisionError( msg )
    perm = [None] *  ncols
    hicol = 1 << ncols
    ah = [ x + (hicol << i) for i,x in enumerate(a) ]
    for i in range(ncols):
        piv = v2(ah[i])
        if piv >= ncols:
            raise ZeroDivisionError( msg )
        perm[piv] = i
        msk = 1 << piv
        for j in range(ncols):
            if ah[j] & msk and i != j:
                ah[j] ^= ah[i]
    return [ ah[perm[i]] >> ncols  for i in range(ncols) ]

def bit_mat_det(a):
    """return the determinant of bit Matrix a."""
    try:
        bit_mat_inverse(a)
        return 1
    except ZeroDivisionError:
        return 0

def bit_mat_orthogonal_complement(a, ncolumns=None):
    """return an orthognal complement of bit Matrix a.

    Optionally, the number of columns of a may be given.
    Here matrix 'a' is considered a matrix of row vectors generating a
    linear subspace W of V = GF(2)**ncolumns. The function returns a
    matrix of row vetors generating the orthognal complement of W with
    respect to the standard Euclidean inner product in V.
    """
    # This can be improved using the ideas in bit_mat_inverse !!
    if ncolumns == None:
         ncolumns = bitlen(reduce(__or__, a, 0))
    piv_a, piv_cols = pivot_binary_low(a)
    for i in range(ncolumns):
        if not i in piv_cols:
            piv_a.append(1 << i)
    ao = bit_mat_transpose(bit_mat_inverse(piv_a), ncolumns)
    return ao[len(piv_cols):]

def bit_mat_rank(a):
    """return the rank of a bit marix a"""
    return len(pivot_binary_high(a)[0])

def bit_mat_random(nrows, ncolumns):
    """return a random bit matrix with given number of rows and columns"""
    a, rmax = [],  ( 1 << ncolumns) - 1
    for i in range(nrows):
         a.append(randint(0, rmax))
    return a

def bit_mat_basis_spanned(basis, dtype = None):
    """return an array of the bitvectors spanned by a basis

    The function returns an array A of length 2**len(basis) containing
    the bit vectors spanned by the basis.

    Entry A[sum(b_i * 2*i)] is sum(b_i * basis[i]), b[i] = 0,1.

    If dtype is not None, a numpy array of the corresponding dtype
    is returned.
    """
    sp = [0]
    for x in basis:
        sp += [ y ^ x for y in sp ]
    if not dtype is None:
        sp = numpy.array(sp, dtype = dtype)
    return sp

def bit_perm_mul(a, *args):
    """ Multiplies 'a' with an arbitrary number of bit permutations.

    Parameter 'a'  may be a bit vector or a list representing a permutation.
    Representation of bit vectors and permutations see comment on module.
    The result returned has the same shape as parameter 'a'.
    """
    if len(args) == 1:
        b = args[0]
        if isinteger(a):
            return reduce(
                   __xor__, [1<<x for i,x in enumerate(b) if a & (1<<i)], 0 )
        else:
            return [ b[x]  for  x in a ]
    if len(args) == 0:
        return a
    return  bit_perm_mul(a, *(args[:-2] + ([args[-1][x] for x in args[-2] ],)))

    raise NotImplementedError("")

def bit_perm_2_mat(a):
    """transforms a permutation into a bit matrix.

    Representation of permutations and bit matrices see comment
    on module.
    """
    if isinteger(a): return a
    return [1 << x for x in a]

def rand_perm(n):
    """returns a list representing a random permutation of n elements"""
    if n == 1: return [0]
    else:
       a = randint(0, n-1)
       p1 = [x for x in range(a,n)+range(a)]
       p2 = [n-1] + rand_perm(n-1)
       return bit_perm_mul(p1,p2)

max_stars_repo_name: Martin-Seysen/mmgroup

max_issues_repo_name: Martin-Seysen/mmgroup

max_forks_repo_name: Martin-Seysen/mmgroup

reason: import numpy,from numpy