text:
import gym
import random
import numpy as np
from pyglet.window import key
from gym.wrappers.monitor import Monitor
import matplotlib.pyplot as plt
import os
import src.reinforcement_learning.env

random.seed(0)  # make results reproducible
advanced_mode = True
record_video = False

if advanced_mode:
    env = gym.make('CarRacingCustomAdvanced-v0',
                   allow_reverse=True,
                   grayscale=1,
                   show_info_panel=1,
                   discretize_actions=None,
                   num_tracks=2,
                   num_lanes=2,
                   num_lanes_changes=6,
                   max_time_out=0,
                   frames_per_state=4,
                   num_obstacles=10)

    discretize = env.discretize_actions
    if discretize is None:
        a = np.array([0.0, 0.0, 0.0])
    else:
        a = np.array([0])

    def key_press(k, mod):
        global restart
        if discretize is None:
            if k == 0xff0d: restart = True
            if k == key.LEFT:  a[0] = -1.0
            if k == key.RIGHT: a[0] = +1.0
            if k == key.UP:    a[1] = +1.0
            if k == key.DOWN:  a[1] = -1.0
            if k == key.SPACE: a[2] = +0.8  # set 1.0 for wheels to block to zero rotation
        elif discretize == "hard":
            if k == 0xff0d: restart = True
            if k == key.LEFT:  a[0] = 1
            if k == key.RIGHT: a[0] = 2
            if k == key.UP:    a[0] = 3
            if k == key.SPACE: a[0] = 4

    def key_release(k, mod):
        if discretize is None:
            if k == key.LEFT and a[0] == -1.0: a[0] = 0
            if k == key.RIGHT and a[0] == +1.0: a[0] = 0
            if k == key.UP:    a[1] = 0
            if k == key.DOWN:  a[1] = 0
            if k == key.SPACE: a[2] = 0
        else:
            a[0] = 0
        if k == key.D:     set_trace()
        if k == key.R:     env.reset()
        if k == key.Z:     env.change_zoom()
        if k == key.G:     env.switch_intersection_groups()
        if k == key.I:     env.switch_intersection_points()
        if k == key.X:     env.switch_xt_intersections()
        if k == key.E:     env.switch_end_of_track()
        if k == key.S:     env.switch_start_of_track()
        if k == key.T:     env.screenshot('./')
        if k == key.Q:     sys.exit()

    env.render()
    if record_video:
        PATH = os.path.abspath('env/') + '/data_samples/core_environments/'
        env = Monitor(env, PATH + 'car_racing_advanced', force=True)
    # env.key_press_fn = key_press
    # env.key_release_fn = key_release

    env.viewer.window.on_key_press = key_press
    env.viewer.window.on_key_release = key_release
    while True:
        env.reset()
        total_reward = 0.0
        steps = 0
        restart = False

        while True:
            if discretize is not None:
                a_tmp = a[0]
            else:
                a_tmp = a
            s, r, done, info = env.step(a_tmp)
            print(s.shape)
            plt.imshow(s.astype(np.uint32))
            # plt.show()
            # print(s.astype(np.uint32))
            total_reward += r
            if steps % 200 == 0 or done:
                # print("\naction " + str(["{:+0.2f}".format(x) for x in a]))
                print("step {} total_reward {:+0.2f}".format(steps, total_reward))
                steps += 1
                print("step {} total_reward {:+0.2f}".format(steps, total_reward))
            steps += 1
            if not record_video:  # Faster, but you can as well call env.render() every time to play full window.
                env.render()
            if done or restart: break

    env.close()

else:
    env = gym.make('CarRacingCustom-v0')
    a = np.array([0.0, 0.0, 0.0])
    def key_press(k, mod):
        global restart
        if k == 0xff0d: restart = True
        if k == key.LEFT:  a[0] = -1.0
        if k == key.RIGHT: a[0] = +1.0
        if k == key.UP:    a[1] = +1.0
        if k == key.DOWN:  a[2] = +0.8   # set 1.0 for wheels to block to zero rotation

    def key_release(k, mod):
        if k == key.LEFT  and a[0] == -1.0: a[0] = 0
        if k == key.RIGHT and a[0] == +1.0: a[0] = 0
        if k == key.UP:    a[1] = 0
        if k == key.DOWN:  a[2] = 0

    env.render()
    env.viewer.window.on_key_press = key_press
    env.viewer.window.on_key_release = key_release
    if record_video:
        PATH = os.path.abspath('env/') + '/data_samples/core_environments/'
        env = Monitor(env, PATH + 'car_racing_default', force=True)
    isopen = True
    while isopen:
        env.reset()
        total_reward = 0.0
        steps = 0
        restart = False
        while True:
            s, r, done, info = env.step(a)
            total_reward += r
            if steps % 200 == 0 or done:
                print("\naction " + str(["{:+0.2f}".format(x) for x in a]))
                print("step {} total_reward {:+0.2f}".format(steps, total_reward))
            steps += 1
            isopen = env.render()
            if done or restart or isopen == False:
                break
    env.close()


text:
# -*- coding: UTF-8 -*-

import os
import re
import numpy as np
import jieba
from gensim.models import KeyedVectors
from tensorflow.python.keras.models import Sequential
from tensorflow.python.keras.layers import Dense, Embedding, LSTM
from tensorflow.python.keras.preprocessing.sequence import pad_sequences
from tensorflow.python.keras.optimizers import Adam
from tensorflow.python.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from sklearn.model_selection import train_test_split

# 导入词向量
cn_word_vecs = KeyedVectors.load_word2vec_format(
    'sgns.zhihu.bigram', binary=False)
ebd_dim = cn_word_vecs[u'我'].shape[0]

POS = os.path.join(os.getcwd(), 'fix_pos')
NEG = os.path.join(os.getcwd(), 'fix_neg')

# 存储所有评价，每例评价为一条string
train_text_raw = []
for file_name in os.listdir(POS):
    with open(os.path.join(POS, file_name), 'r') as f:
        train_text_raw.append(f.read().strip().decode('utf-8'))

for file_name in os.listdir(NEG):
    with open(os.path.join(NEG, file_name), 'r') as f:
        train_text_raw.append(f.read().strip().decode('utf-8'))

# train_tokens是一个长长的list，其中含有4000个小list，对应每一条评价
train_tokens = []
word_lists = []
pattern = ur"[\s+\.\!\/_,$%^*(+\"\']+|[+——！，。？、~@#￥%……&*（）]+"
for text in train_text_raw:
    p_text = re.sub(pattern, "", text)
    cut = jieba.cut(p_text)
    cut_list = []
    word_list = []
    for word in cut:
        word_list.append(word)
        try:
            cut_list.append(cn_word_vecs.vocab[word].index)
        except KeyError:
            cut_list.append(0)
    train_tokens.append(cut_list)
    word_lists.append(word_list)

# 计算出最优长度
num_tokens = np.array([len(tokens) for tokens in train_tokens])
max_tokens = np.mean(num_tokens) + 2 * np.std(num_tokens)
max_tokens = int(max_tokens)

def reverse_tokens(tokens):
    text = ''
    for i in tokens:
        if i != 0:
            text = text + cn_word_vecs.index2word[i]
        else:
            text = text + ' '
    return text

# 构造embeding
num_words = 50000
ebd_matrix = np.zeros((num_words, ebd_dim))
for i in range(num_words):
    ebd_matrix[i, :] = cn_word_vecs[cn_word_vecs.index2word[i]]
ebd_matrix = ebd_matrix.astype('float32')

# 处理不等长的序列
train_pad = pad_sequences(
    train_tokens, maxlen=max_tokens, padding='pre', truncating='pre')
train_pad[train_pad >= num_words] = 0
train_target = np.concatenate((np.ones(2000), np.zeros(2000)))

x_train, x_test, y_train, y_test = train_test_split(
    train_pad, train_target, test_size=0.1, random_state=12)

# 构造模型
model = Sequential()
model.add(
    Embedding(
        num_words,
        ebd_dim,
        weights=[ebd_matrix],
        input_length=max_tokens,
        trainable=False))
model.add(LSTM(units=16, return_sequences=False))
model.add(Dense(1, activation='sigmoid'))
optimizer = Adam(lr=1e-3)
model.compile(
    loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])
model.summary()

# 建立一个权重的存储点
path_checkpoint = 'sentiment_checkpoint.keras'
checkpoint = ModelCheckpoint(
    filepath=path_checkpoint,
    monitor='val_loss',
    verbose=1,
    save_weights_only=True,
    save_best_only=True)

# 定义early stoping如果3个epoch内validation loss没有改善则停止训练
earlystopping = EarlyStopping(monitor='val_loss', patience=3, verbose=1)

# 自动降低learning rate
lr_reduction = ReduceLROnPlateau(
    monitor='val_loss', factor=0.1, min_lr=1e-5, patience=0, verbose=1)
# 定义callback函数
callbacks = [earlystopping, checkpoint, lr_reduction]

model.fit(
    x_train,
    y_train,
    validation_split=0.1,
    epochs=20,
    batch_size=128,
    callbacks=callbacks)

result = model.evaluate(x_test, y_test)
print 'Accuracy:%f' % result[1]

# 测试
def predict_sentiment(text):
    print text
    p_text = re.sub(pattern, "", text)
    cut = jieba.cut(p_text)
    cut_list = []
    for word in cut:
        try:
            cut_list.append(cn_word_vecs.vocab[word].index)
        except KeyError:
            cut_list.append(0)
    tokens_pad = pad_sequences([cut_list],
                               maxlen=max_tokens,
                               padding='pre',
                               truncating='pre')
    tokens_pad[tokens_pad >= num_words] = 0
    result = model.predict(x=tokens_pad)
    coef = result[0][0]
    if coef >= 0.5:
        print '是一例正面评价', 'output=%.2f' % coef
    else:
        print '是一例负面评价', 'output=%.2f' % coef

test_list = [
    '酒店设施不是新的，服务态度很不好', '酒店卫生条件非常不好', '床铺非常舒适', '房间很凉，不给开暖气', '房间很凉爽，空调冷气很足',
    '酒店环境不好，住宿体验很不好', '房间隔音不到位', '晚上回来发现没有打扫卫生', '因为过节所以要我临时加钱，比团购的价格贵'
]
for text in test_list:
    predict_sentiment(text)

ext: py

reason: import numpy


text:
import matplotlib.pyplot as plt
import numpy as np
from scipy.stats import norm
np.random.seed(1)

#%%
BANDIT_PROBABILITIES = [0.2, 0.55, 0.75]
NUM_TRIALS = 10_000

class Bandit:
    def __init__(self, true_mean):
        self.true_mean = true_mean
        # parameters for mu - prior is N(0,1)
        self.m = 0
        self.lambda_ = 1
        self.tau = 1
        self.N = 0

    def pull(self):
        return np.random.randn() / np.sqrt(self.tau) + self.true_mean

    def sample(self):
        return np.random.randn() / np.sqrt(self.lambda_) + self.m

    def update(self, x):
        self.m = (self.tau * x + self.lambda_ * self.m) / (self.tau + self.lambda_)
        self.lambda_ += self.tau
        self.N += 1

def plot(bandits, trial):
    plt.figure()
    x = np.linspace(-3, 6, 200)
    for b in bandits:
        y = norm.pdf(x, b.m, np.sqrt(1. / b.lambda_))
        plt.plot(x, y, label=f"real mean: {b.true_mean:.4f}, num plays: {b.N}")
    plt.title(f"Bandit distributions after {trial} trials")
    plt.legend()
    plt.show()

def experiment():
    bandits = [Bandit(p) for p in BANDIT_PROBABILITIES]
    optimal_j = np.argmax(BANDIT_PROBABILITIES)
    sample_points = [5, 10, 20, 50, 100, 200, 500, 1000, 1500, 1999]
    num_times_explored = 0
    num_times_exploited = 0
    num_optimal = 0
    rewards = np.zeros(NUM_TRIALS)
    for i in range(NUM_TRIALS):
        j = np.argmax([b.sample() for b in bandits])

        # plot the posteriors
        if i in sample_points:
            plot(bandits, i)

        if j==optimal_j:
            num_optimal += 1

        x = bandits[j].pull()
        bandits[j].update(x)
        rewards[i] = x

    cumulative_average = np.cumsum(rewards) / (np.arange(NUM_TRIALS) + 1)
    # plot moving average ctr
    plt.plot(cumulative_average)
    for m in BANDIT_PROBABILITIES:
        plt.plot(np.ones(NUM_TRIALS) * m)
    plt.show()

    print("Number times explored: ",num_times_explored)
    print("Number times exploited : ", num_times_exploited )
    print("Total reward earned: ", rewards.sum())
    print("Win rate: ", rewards.sum()/NUM_TRIALS)
    print("Selected optimal bandit: ", num_optimal)

    return cumulative_average

if __name__ == "__main__":
    experiment()

max_stars_repo_name: chinmaykurade/ab-testing-course

max_issues_repo_name: chinmaykurade/ab-testing-course

max_forks_repo_name: chinmaykurade/ab-testing-course

reason: import numpy,from scipy


text:
from __future__ import division
import numpy as np
from matplotlib import pyplot as plt
from collections import defaultdict

import pyhsmm
from pyhsmm.util.text import progprint_xrange

N = 4
obs_hypparams = dict(mu_0=np.zeros(2),sigma_0=np.eye(2),kappa_0=0.05,nu_0=4)
T = 1000

### generate data

obs_distns = [pyhsmm.distributions.Gaussian(**obs_hypparams) for state in range(N)]
truemodel = pyhsmm.models.HMM(alpha=N,init_state_concentration=1.,obs_distns=obs_distns)

def normalize(mat):
    return np.nan_to_num(mat / mat.sum(1)[:,None])

trans_matrices = [normalize(mat) for mat in
        [
            np.ones((N,N)),
            np.eye(N) + np.diag(0.3*np.ones(N-1),1) + np.diag(0.3*np.ones(N-1),-1),
        ]
    ]

datas, labels = defaultdict(list), defaultdict(list)
for idx, mat in enumerate(trans_matrices):
    truemodel.trans_distn.trans_matrix = mat
    for itr in range(3):
        mydata, mylabels = truemodel.generate(T)
        datas['group%d' % idx].append(mydata)
        labels['group%d' % idx].append(mylabels)

### inference (using true obs_distns)

model = pyhsmm.models.HMMSeparateTrans(alpha=N,init_state_concentration=1.,
        obs_distns=obs_distns)

for group_id, datalist in datas.iteritems():
    for data in datalist:
        model.add_data(group_id=group_id,data=data)

for itr in progprint_xrange(100):
    model.resample_states()
    model.resample_trans_distn()

plt.matshow(model.trans_distns['group1'].trans_matrix)
plt.show()

max_stars_repo_name: bikash/pyhsmm

max_issues_repo_name: bikash/pyhsmm

max_forks_repo_name: bikash/pyhsmm


text:
##################################################################
#Fisica de Semiconductores
#Jeison Ivan Roa Mora
##################################################################
#Importar Librerias
import numpy as np
import matplotlib.pyplot as plt
import scipy.constants   as sc
from matplotlib.widgets import Slider, Button, RadioButtons
from matplotlib.figure import Figure
from matplotlib import colors
from Tkinter import *
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
##################################################################
#Declaracion de variables, ejes y constantes
figure = Figure() #Figura
canvas = FigureCanvas(figure) #Canvas
fig, ax = plt.subplots()
plt.subplots_adjust(left=0.1, bottom=0.25)
Vo=1  #Variable para el valor del potencial
a=1.5*10**-9 #Ancho del potencial en metros
En=np.linspace(0.04,3,1000) #Valores de energia de la particula en eV:
T=[]
R=[]
##################################################################
#Declaracion funcion
def den(Vo,a,E,z):
    k=np.sqrt(2*sc.m_e*abs(Vo-E)*sc.e)/sc.hbar
    x=(Vo**2)/(4.0*E)
    if   z==0:
        return x*(np.sinh(k*a)**2)/(Vo-E)
    elif z==1:
        return x*(np.sin(k*a)**2)/(E-Vo)
    else:
        return (sc.m_e*(a**2))*Vo*sc.e/(2*(sc.hbar)**2)
for E in En:
    if E<=Vo:
        x=1/(1+den(Vo,a,E,0))
    elif E>Vo:
        x=1/(1+den(Vo,a,E,1))
    else:
        x=1/(1+den(Vo,a,E,2))
    T.append(x)
    R.append(1-x)
##################################################################
#PLots
plt.title("Coeficientes de Transmision y Reflexion para Barrera de potencial", ha="center", fontsize=15,color="k", fontweight = 'bold')
plt.ylabel(r'E/Vo', fontsize=16,color='k', fontweight = 'bold')
a0 = 1
f0 = 1.5*10**-9
l, = plt.plot(En,T, lw=2, color='crimson')
ll, = plt.plot(En,R, lw=2, color='k')
dot, = plt.plot(En[300],T[300],'bo', markersize=18)
dot2, = plt.plot(En[300],R[300],'ro', markersize=18)
plt.axis([0, 3, -0.3, 1.3])
axcolor = 'gold'
axfreq = plt.axes([0.1, 0.1, 0.65, 0.03], axisbg=axcolor)
axamp = plt.axes([0.1, 0.15, 0.65, 0.03], axisbg=axcolor)
axpunto=plt.axes([0.1, 0.05, 0.65, 0.03], axisbg=axcolor)
aaa = Slider(axfreq, 'Ancho', 1*10**-9, 2*10**-9, valinit=f0)
samp = Slider(axamp, 'Vo', 0.01, 3.0, valinit=a0)
punto = Slider(axpunto, 'E', 0, 1000, valinit=500)
##################################################################
#Declaracion funcion para refrescar y mostrar la grafica
def update(val):
	a = aaa.val
	Vo=samp.val
	p=punto.val
	T=[]
	R=[]
	for E in En:
		if E<=Vo:
			x=1/(1+den(Vo,a,E,0))
		elif E>Vo:
			x=1/(1+den(Vo,a,E,1))
		else:
			x=1/(1+den(Vo,a,E,2))
		T.append(x)
		R.append(1-x)
	l.set_ydata(T)
	ll.set_ydata(R)
	dot.set_data(En[int(p)],T[int(p)])
	dot2.set_data(En[int(p)],R[int(p)])
	fig.canvas.draw_idle()
aaa.on_changed(update)
samp.on_changed(update)
punto.on_changed(update)
plt.plot(En,T,color=plt.cm.gist_rainbow(50))
plt.plot(En,R,color=plt.cm.gist_rainbow(150))
plt.show()

reason: import numpy,import scipy


text:
import argparse
import os
import pprint

import gym
import numpy as np
import torch
from torch.utils.tensorboard import SummaryWriter

from tianshou.data import Collector, VectorReplayBuffer
from tianshou.env import SubprocVectorEnv
from tianshou.policy import SACPolicy
from tianshou.trainer import offpolicy_trainer
from tianshou.utils import TensorboardLogger
from tianshou.utils.net.common import Net
from tianshou.utils.net.continuous import ActorProb, Critic

def get_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--task', type=str, default="BipedalWalkerHardcore-v3")
    parser.add_argument('--seed', type=int, default=0)
    parser.add_argument('--buffer-size', type=int, default=1000000)
    parser.add_argument('--actor-lr', type=float, default=3e-4)
    parser.add_argument('--critic-lr', type=float, default=1e-3)
    parser.add_argument('--gamma', type=float, default=0.99)
    parser.add_argument('--tau', type=float, default=0.005)
    parser.add_argument('--alpha', type=float, default=0.1)
    parser.add_argument('--auto-alpha', type=int, default=1)
    parser.add_argument('--alpha-lr', type=float, default=3e-4)
    parser.add_argument('--epoch', type=int, default=100)
    parser.add_argument('--step-per-epoch', type=int, default=100000)
    parser.add_argument('--step-per-collect', type=int, default=10)
    parser.add_argument('--update-per-step', type=float, default=0.1)
    parser.add_argument('--batch-size', type=int, default=128)
    parser.add_argument('--hidden-sizes', type=int, nargs='*', default=[128, 128])
    parser.add_argument('--training-num', type=int, default=10)
    parser.add_argument('--test-num', type=int, default=100)
    parser.add_argument('--logdir', type=str, default='log')
    parser.add_argument('--render', type=float, default=0.)
    parser.add_argument('--n-step', type=int, default=4)
    parser.add_argument(
        '--device', type=str, default='cuda' if torch.cuda.is_available() else 'cpu'
    )
    parser.add_argument('--resume-path', type=str, default=None)
    return parser.parse_args()

class Wrapper(gym.Wrapper):
    """Env wrapper for reward scale, action repeat and removing done penalty"""

    def __init__(self, env, action_repeat=3, reward_scale=5, rm_done=True):
        super().__init__(env)
        self.action_repeat = action_repeat
        self.reward_scale = reward_scale
        self.rm_done = rm_done

    def step(self, action):
        rew_sum = 0.0
        for _ in range(self.action_repeat):
            obs, rew, done, info = self.env.step(action)
            # remove done reward penalty
            if not done or not self.rm_done:
                rew_sum = rew_sum + rew
            if done:
                break
        # scale reward
        return obs, self.reward_scale * rew_sum, done, info

def test_sac_bipedal(args=get_args()):
    env = Wrapper(gym.make(args.task))

    args.state_shape = env.observation_space.shape or env.observation_space.n
    args.action_shape = env.action_space.shape or env.action_space.n
    args.max_action = env.action_space.high[0]

    train_envs = SubprocVectorEnv(
        [lambda: Wrapper(gym.make(args.task)) for _ in range(args.training_num)]
    )
    # test_envs = gym.make(args.task)
    test_envs = SubprocVectorEnv(
        [
            lambda: Wrapper(gym.make(args.task), reward_scale=1, rm_done=False)
            for _ in range(args.test_num)
        ]
    )

    # seed
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    train_envs.seed(args.seed)
    test_envs.seed(args.seed)

    # model
    net_a = Net(args.state_shape, hidden_sizes=args.hidden_sizes, device=args.device)
    actor = ActorProb(
        net_a,
        args.action_shape,
        max_action=args.max_action,
        device=args.device,
        unbounded=True
    ).to(args.device)
    actor_optim = torch.optim.Adam(actor.parameters(), lr=args.actor_lr)

    net_c1 = Net(
        args.state_shape,
        args.action_shape,
        hidden_sizes=args.hidden_sizes,
        concat=True,
        device=args.device
    )
    critic1 = Critic(net_c1, device=args.device).to(args.device)
    critic1_optim = torch.optim.Adam(critic1.parameters(), lr=args.critic_lr)

    net_c2 = Net(
        args.state_shape,
        args.action_shape,
        hidden_sizes=args.hidden_sizes,
        concat=True,
        device=args.device
    )
    critic2 = Critic(net_c2, device=args.device).to(args.device)
    critic2_optim = torch.optim.Adam(critic2.parameters(), lr=args.critic_lr)

    if args.auto_alpha:
        target_entropy = -np.prod(env.action_space.shape)
        log_alpha = torch.zeros(1, requires_grad=True, device=args.device)
        alpha_optim = torch.optim.Adam([log_alpha], lr=args.alpha_lr)
        args.alpha = (target_entropy, log_alpha, alpha_optim)

    policy = SACPolicy(
        actor,
        actor_optim,
        critic1,
        critic1_optim,
        critic2,
        critic2_optim,
        tau=args.tau,
        gamma=args.gamma,
        alpha=args.alpha,
        estimation_step=args.n_step,
        action_space=env.action_space
    )
    # load a previous policy
    if args.resume_path:
        policy.load_state_dict(torch.load(args.resume_path))
        print("Loaded agent from: ", args.resume_path)

    # collector
    train_collector = Collector(
        policy,
        train_envs,
        VectorReplayBuffer(args.buffer_size, len(train_envs)),
        exploration_noise=True
    )
    test_collector = Collector(policy, test_envs)
    # train_collector.collect(n_step=args.buffer_size)
    # log
    log_path = os.path.join(args.logdir, args.task, 'sac')
    writer = SummaryWriter(log_path)
    logger = TensorboardLogger(writer)

    def save_best_fn(policy):
        torch.save(policy.state_dict(), os.path.join(log_path, 'policy.pth'))

    def stop_fn(mean_rewards):
        return mean_rewards >= env.spec.reward_threshold

    # trainer
    result = offpolicy_trainer(
        policy,
        train_collector,
        test_collector,
        args.epoch,
        args.step_per_epoch,
        args.step_per_collect,
        args.test_num,
        args.batch_size,
        update_per_step=args.update_per_step,
        test_in_train=False,
        stop_fn=stop_fn,
        save_best_fn=save_best_fn,
        logger=logger
    )

    if __name__ == '__main__':
        pprint.pprint(result)
        # Let's watch its performance!
        policy.eval()
        test_envs.seed(args.seed)
        test_collector.reset()
        result = test_collector.collect(n_episode=args.test_num, render=args.render)
        rews, lens = result["rews"], result["lens"]
        print(f"Final reward: {rews.mean()}, length: {lens.mean()}")

if __name__ == '__main__':
    test_sac_bipedal()

max_stars_repo_name: quangr/tianshou

max_issues_repo_name: quangr/tianshou

max_forks_repo_name: quangr/tianshou


text:
"""
Tests for E and H field calculations
"""
import numpy as np
import miepy

nm = 1e-9

def test_boundary_conditions():
    """verifies the continunity of tangential components of E and H at the surface of a particle"""
    radius = 50*nm

    cluster = miepy.sphere_cluster(position=[[0,0,0]],
                                   radius=radius,
                                   material=miepy.materials. Ag(),
                                   lmax=2,
                                   wavelength=600*nm,
                                   source=miepy.sources.plane_wave.from_string(polarization='y'),
                                   medium=miepy.constant_material(1.2**2))

    theta = 0.3
    phi = 0.3
    eps = .1*nm

    E_out = cluster.E_field(radius + eps, theta, phi, spherical=True)
    E_in = cluster.E_field(radius - eps, theta, phi, spherical=True)

    H_out = cluster.H_field(radius + eps, theta, phi, spherical=True)
    H_in = cluster.H_field(radius - eps, theta, phi, spherical=True)

    assert np.allclose(E_out[1:], E_in[1:], atol=4e-2, rtol=0)
    assert np.allclose(H_out[1:], H_in[1:], atol=4e-2, rtol=0)

max_stars_repo_name: johnaparker/MiePy

max_issues_repo_name: johnaparker/MiePy

max_forks_repo_name: johnaparker/MiePy


text:
import matplotlib.pyplot as plt
import itertools
import numpy as np

# source: http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html#sphx-glr-auto-examples-model-selection-plot-confusion-matrix-py
def plot_confusion_matrix(cmat, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if normalize:
        print("Normalized.")
        cmat = cmat.astype('float') / cmat.sum(axis=1)[:, np.newaxis]

    plt.imshow(cmat, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)
    thresh = cmat.max() / 2.
    for i, j in itertools.product(range(cmat.shape[0]), range(cmat.shape[1])):
        plt.text(j, i, "%.2f"%cmat[i, j],
                 horizontalalignment="center",
                 color="white" if cmat[i, j] > thresh else "black")
    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')


text:
from contextlib import contextmanager
import datetime
import numpy as np
import os
import psycopg2
import psycopg2.pool
import sys

from DBUtils_strings import DBUtils_strings as dbs

try:
    import cPickle as pickle
except:
    import pickle

try:
    import cStringIO as StringIO
except:
    import StringIO

class VALID_COLUMNS:
    RUN_CONFIG = (
        "id",
        "networks_id",
        "trails_id",
        "mutate_id",
        "selection_id",
        "generations",
        "population",
        "moves_limit",
        "sel_tourn_size",
        "p_mutate",
        "p_crossover",
        "weight_min",
        "weight_max",
        "lambda",
        "variations_id",
        "algorithm_ver",
        "RowNumber")

class DBUtils:
    FILTERS_IDS = [
        "networks_id",
        "trails_id",
        "mutate_id",
        "selection_id",
        "generations",
        "population",
        "moves_limit",
        "sel_tourn_size",
        "p_mutate",
        "p_crossover",
        "weight_min",
        "weight_max",
        "lambda",
        "variations_id",
        "algorithm_ver"]

    FILTERS_STRINGS = [
        "Networks",
        "Trails",
        "Mutate ID",
        "Selection",
        "Generations",
        "Population (" u"\u03BC" ")",
        "Moves Limit",
        "Tournament Size",
        "P(mutate)",
        "P(Crossover)",
        "Min. Weight",
        "Max. Weight",
        u"\u03BB",
        "Variation",
        "Algorithm"]

    def __init__(self, dsn=None):

        if not dsn:
            dsn = (
                "host={0} dbname={1} user={2} password={3} port={4}".format(
                    os.environ.get("PGHOST", "localhost"),
                    os.environ.get("PGDATABASE", "jmoles"),
                    os.environ.get("PGUSER", "jmoles"),
                    os.environ.get("PGPASSWORD", "password"),
                    os.environ.get("PGPORT", 5432)))

        self.__pool        = psycopg2.pool.SimpleConnectionPool(1, 10, dsn)

    @contextmanager
    def __getCursor(self):
        con = self.__pool.getconn()
        try:
            yield con.cursor()
        finally:
            self.__pool.putconn(con)

    def getNetworks(self):
        return self.__genericDictGet("SELECT id, name FROM networks")

    def getTrails(self):
        ret_dict = {}

        with self.__getCursor() as curs:
            curs.execute("""SELECT id, name, moves, init_rot, trail_data
                FROM trails""")
            for idx, name, moves, rot, data in curs.fetchall():
                temp_dict = {
                    'name' : name,
                    'moves' : moves,
                    'rot' : rot,
                    'data' : data}
                ret_dict[idx] = temp_dict

        return ret_dict

    def getMutates(self):
        return self.__genericDictGet("SELECT id, name FROM mutate")

    def getSelect(self):
        return self.__genericDictGet("SELECT id, name FROM selection")

    def __genericDictGet(self, query):
        ret_dict = {}

        with self.__getCursor() as curs:
            curs.execute(query)
            for idx, name in curs.fetchall():
                ret_dict[idx] = name

        return ret_dict

    def getTrailSQL(self):
        with self.__getCursor() as curs:
            curs.execute("""SELECT * FROM trails;""")

            curs_results = curs.fetchall()

        return curs_results

    def getTrailData(self, trailID):
        with self.__getCursor() as curs:
            curs.execute("""SELECT trail_data, name, init_rot, moves
                FROM trails
                WHERE id=%s;""", (trailID, ))

            curs_results = curs.fetchall()[0]

        return (
            np.matrix(curs_results[0]),
            curs_results[1],
            curs_results[2],
            curs_results[3])

    def fetchRunGenerations(self, run_id):

        # TODO: Need to make a view to properly handle this function.

        with self.__getCursor() as curs:
            curs.execute("""SELECT run_id, generation, runtime,
                food_min, food_max, food_avg, food_std,
                moves_min, moves_max, moves_avg, moves_std,
                moves_left, moves_right, moves_forward, moves_none
                FROM generations
                WHERE run_id IN %s;""", (tuple(run_id), ) )

            ret_val  = {}

            for record in curs:

                curr_run_id        = record[0]
                curr_gen           = record[1]

                food_d             = {}
                food_d["min"]      = record[3]
                food_d["max"]      = record[4]
                food_d["avg"]      = record[5]
                food_d["std"]      = record[6]

                move_d             = {}
                move_d["min"]      = record[7]
                move_d["max"]      = record[8]
                move_d["avg"]      = record[9]
                move_d["std"]      = record[10]
                move_d["left"]     = record[11]
                move_d["right"]    = record[12]
                move_d["forward"]  = record[13]
                move_d["none"]     = record[14]

                # TODO: Need to double index this table here. Once for run,
                # and then once for each generation.
                curr_dict          = { "food" : food_d, "moves" : move_d }

                if curr_run_id in ret_val:
                    ret_val[curr_run_id][curr_gen] = curr_dict
                else:
                    init_dict = {curr_gen : curr_dict }
                    ret_val[curr_run_id] = init_dict

        return ret_val

    def fetchRunInfo(self, run_id):
        if isinstance(run_id, int):
            run_id = (run_id, )

        with self.__getCursor() as curs:
            curs.execute("""SELECT run.id, trails_id, networks_id, mutate_id,
                host_configs_id, run_date, runtime, hostname, generations,
                population, moves_limit, sel_tourn_size, p_mutate, p_crossover,
                weight_min, weight_max, debug, run_config.id, selection_id,
                lambda, variations_id, algorithm_ver
                FROM run
                INNER JOIN run_config
                ON run.run_config_id = run_config.id
                WHERE run.id IN %s;""", (tuple(run_id), ) )

            ret_val = {}

            for record in curs:
                curr_dict                    = {}

                this_run_id                  = record[0]
                curr_dict["trails_id"]       = record[1]
                curr_dict["networks_id"]     = record[2]
                curr_dict["mutate_id"]       = record[3]
                curr_dict["host_configs_id"] = record[4]
                curr_dict["run_date"]        = record[5]
                curr_dict["runtime"]         = record[6]
                curr_dict["hostname"]        = record[7]
                curr_dict["generations"]     = record[8]
                curr_dict["population"]      = record[9]
                curr_dict["moves_limit"]     = record[10]
                curr_dict["sel_tourn_size"]  = record[11]
                curr_dict["p_mutate"]        = record[12]
                curr_dict["p_crossover"]     = record[13]
                curr_dict["weight_min"]      = record[14]
                curr_dict["weight_max"]      = record[15]
                curr_dict["debug"]           = record[16]
                curr_dict["run_config_id"]   = record[17]
                curr_dict["selection_id"]    = record[18]
                curr_dict["lambda"]          = record[19]
                curr_dict["variations_id"]   = record[20]
                curr_dict["algorithm_ver"]   = record[21]

                ret_val[this_run_id]       = curr_dict

        return ret_val

    @staticmethod
    def __build_where_filters(filters=None, table="run_config"):
        """ Takes a given set of filters and builds a SQL ready
        WHERE statement (or statements for multiple filters) and returns
        a string ready for variable substitution in psycopg2 cursor
        execute.

        Returns:
            str. A string of the WHERE part of query.
        """

        start_filter = True
        filter_str = ""

        if table == "run_config":
            valid_cols = VALID_COLUMNS.RUN_CONFIG

        for curr_key in filters.iterkeys():
            this_s = ""

            if start_filter:
                this_s += "WHERE "
                start_filter = False
            else:
                this_s += "AND "

            if curr_key in valid_cols:
                filter_str += this_s + "{0} = %s\n".format(curr_key)

        return filter_str

    def table_listing(
            self,
            filters=None):

        if filters == None:
            filters = {"generations" : 200}

        where_str = DBUtils.__build_where_filters(filters)

        # Build the base query string with the sort column plugged in.
        data_query_str = """SELECT id, trails_id, networks_id, selection_id,
            generations, population, moves_limit, sel_tourn_size, mutate_id,
            p_mutate, p_crossover, weight_min, weight_max
            FROM   run_config
            {0};""".format(where_str)

        # Run the query and get the data table.
        with self.__getCursor() as curs:
            curs.execute(data_query_str, filters.values())

            run_config_l = curs.fetchall()

        ret_val = []

        # Now, add the run IDs matching the run IDs.
        for curr_run_info in run_config_l:
            result = self.getRunsWithConfigID(curr_run_info[0])

            joined_list = curr_run_info + (list(result), )

            ret_val.append(joined_list)

        return ret_val

    def build_filter_options(self, filters=None):
        """ Returns a list with options for filtering based of data
        in the run_config table."""

        where_str = DBUtils.__build_where_filters(filters)

        query_str = """SELECT
        networks_id, trails_id, mutate_id, selection_id, generations, population,
        moves_limit, sel_tourn_size, p_mutate, p_crossover, weight_min,
        weight_max, lambda, variations_id, algorithm_ver
        FROM run_config
        {0};""".format(where_str)

        with self.__getCursor() as curs:
            curs.execute(query_str, filters.values())

            curs_results = curs.fetchall()

        ret_val = [
            list(set(xx)) for xx in zip(
                *[list(yy) for yy in curs_results][::-1])]

        return ret_val

    def getRunsWithConfigID(self, config_id):
        """ Takes a configuration id and returns all of the run_ids that
        were ran with this configuration.

        Returns:
           list. A list of run_id (as int) that have same configuration_id.

        """
        query_str = """SELECT ARRAY(SELECT id
        FROM run
        WHERE run_config_id = %s);"""

        with self.__getCursor() as curs:
            curs.execute(query_str, (config_id, ))

            ret_val = curs.fetchall()[0][0]

        return ret_val

    def fetchConfigInfo(self, config_id):
        """ Takes a config_id and returns a dictionary with the
        parameters used on this run.

        Returns:
            dict. Of the configuration used on this run.

        """

        with self.__getCursor() as curs:
            curs.execute("""SELECT *
                FROM run_config_full
                WHERE id = %s;""", (config_id, ) )

            result = curs.fetchall()[0]

            curr_dict                    = {}

            curr_dict["trails_id"]       = result[1]
            curr_dict["networks_id"]     = result[2]
            curr_dict["mutate_id"]       = result[3]
            curr_dict["selection_id"]    = result[4]
            curr_dict["generations"]     = result[5]
            curr_dict["population"]      = result[6]
            curr_dict["moves_limit"]     = result[7]
            curr_dict["sel_tourn_size"]  = result[8]
            curr_dict["p_mutate"]        = result[9]
            curr_dict["p_crossover"]     = result[10]
            curr_dict["weight_min"]      = result[11]
            curr_dict["weight_max"]      = result[12]
            curr_dict["network_name"]    = result[13]
            curr_dict["network_dl_len"]  = result[14]
            curr_dict["trail_name"]      = result[15]
            curr_dict["trail_moves"]     = result[16]
            curr_dict["trail_init_rot"]  = result[17]
            curr_dict["trail_data"]      = np.matrix(result[18])
            curr_dict["select_name"]     = result[19]
            curr_dict["mutate_name"]     = result[20]
            curr_dict["lambda"]          = result[21]
            curr_dict["variations_id"]   = result[22]
            curr_dict["algorithm_ver"]   = result[23]
            curr_dict["variations_name"] = result[24]

            curr_dict["max_food"] = np.bincount(np.squeeze(np.asarray(
                curr_dict["trail_data"].flatten())))[1]

        return curr_dict

    def fetch_run_config_sweep(self, config_id, sweep_type="dl_length", id_filters=None):
        # Means query string returns two values (x, y) rather than just one (x)
        is_3d = False

        # Determine the query string to use.
        if sweep_type == "dl_length":
            query_s = dbs.DL_LENGTH_SWEEP
        elif sweep_type == "hidden":
            query_s = dbs.HIDDEN_COUNT_SWEEP
        elif sweep_type == "dl_length_hidden":
            query_s = dbs.DL_LENGTH_HIDDEN_SWEEP
            is_3d = True
        elif sweep_type == "p_mutate":
            query_s = dbs.P_MUTATE_SWEEP
        elif sweep_type == "p_crossover":
            query_s = dbs.P_CROSSOVER_SWEEP
        elif sweep_type == "selection":
            query_s = dbs.SELECTION_SWEEP
        elif sweep_type == "moves_limit":
            query_s = dbs.MOVES_LIMIT_SWEEP
        elif sweep_type == "population":
            query_s = dbs.POPULATION_SWEEP
        elif sweep_type == "generation":
            query_s = dbs.GENERATION_SWEEP
        elif sweep_type == "p_mutate_crossover":
            query_s = dbs.P_MUTATE_CROSSOVER_SWEEP
            is_3d = True
        elif sweep_type == "tournament":
            query_s = dbs.TOURNAMENT_SWEEP
        elif sweep_type == "lambda":
            query_s = dbs.LAMBDA_SWEEP
        else:
            # Invalid sweep specified.
            return

        # Create the tuple of values to place in cursor.
        if (sweep_type == "dl_length" or sweep_type == "hidden" or
            sweep_type == "dl_length_hidden"):
            curs_tuple = ((config_id, ) * (17 - is_3d))
        elif sweep_type == "generation":
            curs_tuple = ((config_id, ) * (13 - is_3d))
        else:
            curs_tuple = ((config_id, ) * (14 - is_3d))

        if (sweep_type == "dl_length"):
            # Manually do these queries a different method.

            # Get the run configuration IDs and dl_lengths.
            # Then filter the config IDs to get run IDs.
            # Finally, find the max food for each run ID.
            with self.__getCursor() as curs:
                curs.execute(dbs.DL_LENGTH_SWEEP_CONFIG_IDS, curs_tuple)
                config_ids_length_t = curs.fetchall()

            # Create a dictionary to later lookup the length for a given
            # configuration id.
            len_by_cnf_id_d = dict(config_ids_length_t)

            # Get all of the run_ids we need to search by.
            with self.__getCursor() as curs:
                curs.execute("""SELECT run.id, run.run_config_id
                    FROM run
                    WHERE run_config_id IN %s;""",
                    (tuple(len_by_cnf_id_d.keys()),)
                )
                run_ids_config_ids_t = curs.fetchall()

            run_ids = [x for x, _ in run_ids_config_ids_t]
            config_ids_per_run = [x for _, x in run_ids_config_ids_t]

            # Create a dictionary to lookup the corresponding config id
            # for the provided run id.
            config_id_dict = {x : [] for x in run_ids}

            for run_id, cnf_id in run_ids_config_ids_t:
                config_id_dict[run_id] = cnf_id

            # Create the return value dictionary with keys
            # as the delay line length.
            ret_val = {x : [] for x in len_by_cnf_id_d.values()}

            # Now, find the max food and min moves for each run_id.
            with self.__getCursor() as curs:
                curs.execute("""SELECT run_id, food_max, moves_min
                    FROM single_run_bests
                    WHERE run_id IN %s""", (tuple(run_ids), )
                )

                stats_by_run_id_d = curs.fetchall()

            # Go through each record and prepare the tuple of data.
            # Then, find the delay line length corresponding to this
            # run id and insert it into the return value.
            for run_id, food_max, moves_min in stats_by_run_id_d:

                data_t = (food_max, moves_min, run_id)

                # Find the delay line length for this run and insert
                # it into the return value.
                dl_len = len_by_cnf_id_d[config_id_dict[run_id]]

                ret_val[dl_len].append(data_t)

            return ret_val

        else:
            with self.__getCursor() as curs:
                curs.execute(query_s, curs_tuple)

                ret_val = {}

                for record in curs:

                    # Offset the records by 1 if it is 3D fetch.
                    sweep_idx_outer = record[0]
                    sweep_idx_inner = record[0 + is_3d]
                    food_max = record[1 + is_3d]
                    moves_min = record[2 + is_3d]
                    run_id = record[3 + is_3d]

                    if  len(record) > (4 + is_3d):
                        temp_tuple = (food_max, moves_min, run_id,
                            record[4 + is_3d])
                    else:
                        temp_tuple = (food_max, moves_min, run_id)

                    if is_3d:
                        if not ret_val.has_key(sweep_idx_outer):
                            ret_val[sweep_idx_outer] = {}

                        if not ret_val[sweep_idx_outer].has_key(
                                sweep_idx_inner):
                            ret_val[sweep_idx_outer][sweep_idx_inner] = []

                        ret_val[sweep_idx_outer][sweep_idx_inner].append(
                            temp_tuple)

                    else:
                        if not ret_val.has_key(sweep_idx_outer):
                            ret_val[sweep_idx_outer] = []

                        ret_val[sweep_idx_outer].append(temp_tuple)

            return ret_val

    def fetchConfigRunsInfo(self, config_id):
        """ Generates a table with run_id, run_date, best food, and
        best moves with a provided config_id. Returns the results a list
        containing a dictionary of the items.

        Returns:
           list. A list containing dictionaries of the items above with
           keys of id, run_date, food, moves.

        """
        with self.__getCursor() as curs:
            curs.execute("""SELECT
                run.id,
                run.run_date,
                run.debug,
                MAX(generations.food_max),
                MIN(moves_min)
                FROM run
                INNER JOIN generations
                ON run.id = generations.run_id
                WHERE run.id IN (
                    SELECT id
                    FROM run
                    WHERE run_config_id = %s)
                GROUP BY run.id
                ORDER BY run.id;""",
                (config_id, ))

            ret_val = []

            for record in curs:
                temp_dict              = {}
                temp_dict["id"]        = record[0]
                temp_dict["run_date"]  = record[1]
                temp_dict["debug"]     = record[2]
                temp_dict["food"]      = record[3]
                temp_dict["moves"]     = record[4]

                ret_val.append(temp_dict)

        return ret_val

max_stars_repo_name: jmoles/trail-viewer

max_issues_repo_name: jmoles/trail-viewer

max_forks_repo_name: jmoles/trail-viewer


text:
"""
Copyright © 2021 United States Government as represented
by the Administrator of the National Aeronautics and Space Administration.
No copyright is claimed in the United States under Title 17, U.S. Code. All Other Rights Reserved.
"""

"""
common functions for sampling spacetime data with "spacecraft" trajectories
"""

import h5py as h5
import numpy as np
import scipy.ndimage as ndi

def find_pixel_bounds(arr, loc):
    if isinstance(loc, float) or isinstance(loc, int):
        return find_pixel_bounds_single(arr, loc)
    else:
        return find_pixel_bounds_arr(arr, loc)

def find_pixel_bounds_arr(arr, locs):
    idxs = []
    for loc in locs:
        idxs.append(find_pixel_bounds_single(arr,loc))

    return idxs

def find_pixel_bounds_single(arr, loc):
    """
    arr: sorted array of values
    loc: single value
    Given loc and arr, return fractional index of loc in arr
    E.g. arr = [1,2,3]; loc = 1.5 -> idx = 0.5
    If loc is outside bounds of arr, return -1 if loc < arr[0] or len(arr) if loc > arr[-1]
    """
    try:
        idx_r = np.where(arr >= loc)[0][0]
    except IndexError:
        idx_r = len(arr)

    try:
        idx_l = np.where(arr <= loc)[0][-1]
    except IndexError:
        idx_l = -1

    if idx_l == -1:
        return idx_l
    elif idx_r == len(arr):
        return idx_r
    elif idx_l == idx_r:
        return idx_l
    else:
        return idx_l + (loc-arr[idx_l])/(arr[idx_r]-arr[idx_l])

def transform_to_pixel_coords(space, time, x, t):
    """
    transforms input (x,t) into pixel coordinates (fractional indexes)
    based on input space/time arrays
    """
    idx_x = find_pixel_bounds(space,x)
    idx_t = find_pixel_bounds(time,t)

    return idx_x, idx_t

def data_along_trajectory(space, time, spacetime_data, xt,tt, order=3):
    """
    Interpolates given data to given trajectory

    space,time : input arrays of space/time coordinates
    spacetime_data: input data corresponding to space/time coordinates
    xt, tt: trajectory space/time coordinates

    returns data corresponding to trajectory space/time
    """
    idx_x, idx_t = transform_to_pixel_coords(space, time, xt,tt)

    return ndi.map_coordinates(spacetime_data, [idx_x, idx_t], order=order)

def linear_data(x,y,npts):
    """
    Returns column stack of linear (x,y) data; line is from (x1,y1) to (x2,y2)
    x: (x1,x2)
    y: (y1,y2)
    npts: number of points in returned data
    """
    x1,x2 = x
    y1, y2 = y
    slope = (y2-y1)/(x2-x1) if x2-x1 != 0. else 0.
    x_sp = np.linspace(x1,x2,npts)

    y_sp = y1+slope*(x_sp-x1)

    return np.column_stack((x_sp,y_sp))

def spacecraft_sod_data(space, time, st_data, traj):
    """
    returns data sampled from `st_data` along trajectory `traj`.
    idx 0: density
    idx 1: velocity
    idx 2: pressure
    """
    dat_0 = data_along_trajectory(space, time, st_data[0], traj[:,0], traj[:,1])
    dat_1 = data_along_trajectory(space, time, st_data[1], traj[:,0], traj[:,1])
    dat_2 = data_along_trajectory(space, time, st_data[2], traj[:,0], traj[:,1])

    return np.column_stack((dat_0, dat_1, dat_2))

def load_sp_sod_data(fle):
    """
    Given filename `fle` corresponding to Sod shocktube spacetime data,
    return:

    space: 1D spatial coordinates of shocktube
    time: 1D temporal coordinates of shocktube
    sc_1 -> sc_4 : four spacecraft trajectories (x,t) through spacetime
    st_data: raw plasma variables from shocktube
    st_input: stacked spacecraft space/time data for input into NN
    data_input: stacked plasma data associated with st_input for input into NN

    NOTE: manually change definitions of sc_1 -> sc_4 to adjust trajectories
    """
    # spacetime data
    euler_st = h5.File(fle, 'r')

    space = euler_st['x'][:]
    time = euler_st['time'][:]
    st_data = np.array(euler_st['U'])
    euler_st.close()

    nx = space.shape[0]
    nt = time.shape[0]

    sp_min = space[0]
    sp_max = space[-1]

    t_min = time[0]
    t_max = time[-1]

    # sample from st data

    # #semi random
    # sc_1 = linear_data((0.002,0.998),(0.05,0.2),150)
    # sc_2 = linear_data((0.002,0.75),(0.002,0.248),150)
    # sc_3 = linear_data((0.65,0.35),(0.002,0.248),150)
    # sc_4 = linear_data((0.998,0.0022),(0.12,0.248),150)

    #parallel over (0,1)X(0,0.25) (x,t) domain
    sc_1 = linear_data((0.002,0.478),(0.175,0.248),75)
    sc_2 = linear_data((0.002,0.998),(0.045,0.2),75)
    sc_3 = linear_data((0.002,0.902),(0.11,0.248),75)
    sc_4 = linear_data((0.1306,0.998),(0.002,0.135),75)

    #interpolate from data
    U_sc1 = spacecraft_sod_data(space, time, st_data, sc_1)
    U_sc2 = spacecraft_sod_data(space, time, st_data, sc_2)
    U_sc3 = spacecraft_sod_data(space, time, st_data, sc_3)
    U_sc4 = spacecraft_sod_data(space, time, st_data, sc_4)

    #stack and return
    st_input = np.concatenate((sc_1, sc_2, sc_3, sc_4))
    data_input = np.concatenate((U_sc1, U_sc2, U_sc3, U_sc4))

    return space,time,sc_1,sc_2,sc_3,sc_4,st_data,st_input, data_input

def load_one_sp_sod_data(fle):
    """
    Given filename `fle` corresponding to Sod shocktube spacetime data,
    return:

    space: 1D spatial coordinates of shocktube
    time: 1D temporal coordinates of shocktube
    sc_1: one spacecraft trajectory (x,t) through spacetime
    st_data: raw plasma variables from shocktube
    st_input: spacecraft space/time data for input into NN
    data_input: plasma data associated with st_input for input into NN

    NOTE: manually adjust sc_1 to change trajectory

    """
    # spacetime data
    euler_st = h5.File(fle, 'r')

    space = euler_st['x'][:]
    time = euler_st['time'][:]
    st_data = np.array(euler_st['U'])
    euler_st.close()

    nx = space.shape[0]
    nt = time.shape[0]

    sp_min = space[0]
    sp_max = space[-1]

    t_min = time[0]
    t_max = time[-1]

    # sample from st data

    #single diagnoal over (0,1)X(0,0.25) (x,t) domain
    sc_1 = linear_data((0.002,0.998),(0.02,0.248),75)

    #interpolate from data
    U_sc1 = spacecraft_sod_data(space, time, st_data, sc_1)

    #stack and return
    st_input = sc_1
    data_input = U_sc1

    return space,time,sc_1,st_data,st_input, data_input

def load_two_sp_sod_data(fle):
    """
    Given filename `fle` corresponding to Sod shocktube spacetime data,
    return:

    space: 1D spatial coordinates of shocktube
    time: 1D temporal coordinates of shocktube
    sc_1 -> sc_2 : two spacecraft trajectories (x,t) through spacetime
    st_data: raw plasma variables from shocktube
    st_input: stacked spacecraft space/time data for input into NN
    data_input: stacked plasma data associated with st_input for input into NN

    NOTE: manually adjust sc_1/sc_2 to change trajectories
    """
    # spacetime data
    euler_st = h5.File(fle, 'r')

    space = euler_st['x'][:]
    time = euler_st['time'][:]
    st_data = np.array(euler_st['U'])
    euler_st.close()

    nx = space.shape[0]
    nt = time.shape[0]

    sp_min = space[0]
    sp_max = space[-1]

    t_min = time[0]
    t_max = time[-1]

    # sample from st data

    #single diagnoal over (0,1)X(0,0.25) (x,t) domain
    sc_1 = linear_data((0.002,0.998),(0.02,0.248),75)
    sc_2 = linear_data((0.4,0.998),(0.002,0.14),75)

    #interpolate from data
    U_sc1 = spacecraft_sod_data(space, time, st_data, sc_1)
    U_sc2 = spacecraft_sod_data(space, time, st_data, sc_2)

    #stack and return
    st_input = np.concatenate((sc_1, sc_2))
    data_input = np.concatenate((U_sc1, U_sc2))

    return space,time,sc_1,sc_2,st_data,st_input, data_input

def spacecraft_mhd_data(space, time, st_data, traj):
    """
    returns data sampled from `st_data` along trajectory `traj`.
    idx 0: density
    idx 1: v_x
    idx 2: v_y
    idx 3: v_z
    idx 4: pressure
    idx 5: b_x
    idx 6: b_y
    idx 7: b_z
    """
    dat_0 = data_along_trajectory(space, time, st_data[0], traj[:,0], traj[:,1])
    dat_1 = data_along_trajectory(space, time, st_data[1], traj[:,0], traj[:,1])
    dat_2 = data_along_trajectory(space, time, st_data[2], traj[:,0], traj[:,1])
    dat_3 = data_along_trajectory(space, time, st_data[3], traj[:,0], traj[:,1])
    dat_4 = data_along_trajectory(space, time, st_data[4], traj[:,0], traj[:,1])
    dat_5 = data_along_trajectory(space, time, st_data[5], traj[:,0], traj[:,1])
    dat_6 = data_along_trajectory(space, time, st_data[6], traj[:,0], traj[:,1])
    dat_7 = data_along_trajectory(space, time, st_data[7], traj[:,0], traj[:,1])

    return np.column_stack((dat_0, dat_1, dat_2, dat_3, dat_4, dat_5, dat_6, dat_7))

def load_sp_mhd_data(fle, problem, parallel=False):
    """
    Given filename `fle` corresponding to shocktube spacetime data,
    `problem` = 'briowu' or 'rj4a'
    return:

    space: 1D spatial coordinates of shocktube
    time: 1D temporal coordinates of shocktube
    sc_1 -> sc_4 : four spacecraft trajectories (x,t) through spacetime
    st_data: raw plasma variables from shocktube
    st_input: stacked spacecraft space/time data for input into NN
    data_input: stacked plasma data associated with st_input for input into NN

    NOTE: manually adjust sc_1 - sc_4 to change trajectories
    `parallel` keyword is used for my convenience
    """
    # spacetime data
    mhd_st = h5.File(fle, 'r')

    space = mhd_st['x'][:]
    time = mhd_st['t'][:]
    st_data = np.array(mhd_st['U'])
    mhd_st.close()

    nx = space.shape[0]
    nt = time.shape[0]

    sp_min = space[0]
    sp_max = space[-1]

    t_min = time[0]
    t_max = time[-1]

    # sample from st data

    if(problem == 'briowu'):
        if parallel:
            #parallel over (0,1)X(0,0.25) (x,t) domain
            sc_1 = linear_data((0.002,0.478),(0.175,0.248),75)
            sc_2 = linear_data((0.002,0.998),(0.045,0.2),75)
            sc_3 = linear_data((0.002,0.902),(0.11,0.248),75)
            sc_4 = linear_data((0.1306,0.998),(0.002,0.135),75)
        else:
            #semi random
            sc_1 = linear_data((0.002,0.998),(0.05,0.2),150)
            sc_2 = linear_data((0.002,0.75),(0.002,0.248),150)
            sc_3 = linear_data((0.65,0.35),(0.002,0.248),150)
            sc_4 = linear_data((0.998,0.0022),(0.12,0.248),150)

    elif(problem == 'rj4a'):
        # #semi random (max time = 0.18 for rj 4a)
        sc_1 = linear_data((0.002,0.998),(0.05,0.12),75)
        sc_2 = linear_data((0.002,0.75),(0.002,0.178),75)
        sc_3 = linear_data((0.65,0.35),(0.002,0.178),75)
        sc_4 = linear_data((0.998,0.0022),(0.12,0.178),75)

    #interpolate from data
    U_sc1 = spacecraft_mhd_data(space, time, st_data, sc_1)
    U_sc2 = spacecraft_mhd_data(space, time, st_data, sc_2)
    U_sc3 = spacecraft_mhd_data(space, time, st_data, sc_3)
    U_sc4 = spacecraft_mhd_data(space, time, st_data, sc_4)

    #stack and convert to tf tensors
    st_input = np.concatenate((sc_1, sc_2, sc_3, sc_4))
    data_input = np.concatenate((U_sc1, U_sc2, U_sc3, U_sc4))

    return space,time,sc_1,sc_2,sc_3,sc_4,st_data,st_input, data_input

def add_noise(data, std=0.03):
    """
    Given `data`, add random Gaussian noise with standard deviation `std`
    """
    rng = np.random.default_rng(42)
    noise = rng.normal(0.0, std, size=data.shape) #roughly -3*std < noise < 3*std

    # enforce positive density/pressure (set to zero)
    noisy_data = data+noise
    noisy_data[:,0] = np.clip(noisy_data[:,0], a_min=0., a_max=None)
    noisy_data[:,4] = np.clip(noisy_data[:,4], a_min=0., a_max=None)

    return noisy_data


text:
'''
Created on Oct 3, 2017

@author: zli
'''
import os, argparse, sys, json, math
import numpy as np
import laspy
import copy

# Scanalyzer -> MAC formular
# Mx = ax + bx * Gx + cx * Gy
# My = ay + by * Gx + cy * Gy
ay = 3659974.971; by = 1.0002; cy = 0.0078
ax = 409012.2032; bx = 0.009; cx = -0.9986

def main():

    las_path = '/media/zli/data/Terra/sample_files/scanner3dTop/2017-07-03/2017-07-03__01-11-05-770/scanner3DTop_L1_ua-mac_2017-07-03__01-11-05-770_merged.las'
    out_path = '/media/zli/data/Terra/sample_files/scanner3dTop/2017-07-03/2017-07-03__01-11-05-770/fixed_header_mac.las'
    file_id = 0
    json_path = '/media/zli/data/Terra/sample_files/scanner3dTop/2017-07-03/2017-07-03__01-11-05-770/23154db0-9771-4955-b4e4-70903278486e_metadata.json'

    geo_referencing_las_for_eachpoint_in_mac(las_path, out_path, file_id, json_path)

    return

# This function add geo-reference offset(UTM ZONE 12) to the las header
def geo_referencing_las(input_las_file, output_las_file, file_id, json_file):

    # open las file
    inFile = laspy.file.File(input_las_file, mode='r')

    # get input header
    input_header = inFile.header

    # create output header
    output_header = copy.copy(input_header)
    origin_coord = get_origin_utm_coordinate(json_file, True)

    output_header.x_offset = origin_coord[0]*1000
    output_header.y_offset = origin_coord[1]*1000
    output_header.z_offset = origin_coord[2]*1000

    # save as new las file
    output_file = laspy.file.File(output_las_file, mode='w', header=output_header)
    output_file.points = inFile.points
    output_file.close()

    return

# This function translate each point to there MAC coordinate without any modify in header
def geo_referencing_las_for_eachpoint_in_mac(input_las_file, output_las_file, file_id, json_file):

    # open las file
    inFile = laspy.file.File(input_las_file, mode='r')

    # output handle
    new_header = copy.copy(inFile.header)
    src_header = inFile.header
    new_header.scale = [0.01,0.01,0.01]
    output_file = laspy.file.File(output_las_file, mode='w', header=new_header)

    origin_coord = get_origin_utm_coordinate(json_file, True)

    #output_file.points = inFile.points
    # do the translating to each point
    output_file.X = inFile.X + long(math.floor(origin_coord[0]*100000))
    output_file.Y = inFile.Y + long(math.floor(origin_coord[1]*100000))
    output_file.Z = inFile.Z + long(math.floor(origin_coord[2]*100000))

    output_file.close()

    return

def get_origin_utm_coordinate(json_file, use_mac=False):

    metadata = lower_keys(load_json(json_file))

    scan_start_position = get_position(metadata) # scan_start_position: gantry position + camera position

    # all the offset has been declare @: https://github.com/terraref/reference-data/issues/44#issuecomment-299336814
    x = scan_start_position[0] + 0.082 # x direction +82mm to the north
    z = (scan_start_position[2] + 0.3) - 3.445 # z direction +3.445m, +0.3 for gantry z 0 is 0.3 meters higher than the ground

    # y is only relate to the origin offset
    if scan_start_position[1]==0:
        y = 3.45  # positive scan, +3.45m y offset
    else:
        y = 25.771 # negative scan, +25.771m y offset

    if use_mac:
        Mx, My = scanalyzer2MAC(x, y)
        return [Mx, My, z]
    else:
        return [x, y, z]

def scanalyzer2MAC(Gx, Gy):

    Mx = ax + bx * Gx + cx * Gy
    My = ay + by * Gx + cy * Gy

    return Mx, My

def load_json(meta_path):
    try:
        with open(meta_path, 'r') as fin:
            return json.load(fin)
    except Exception as ex:
        fail('Corrupt metadata file, ' + str(ex))

def lower_keys(in_dict):
    if type(in_dict) is dict:
        out_dict = {}
        for key, item in in_dict.items():
            out_dict[key.lower()] = lower_keys(item)
        return out_dict
    elif type(in_dict) is list:
        return [lower_keys(obj) for obj in in_dict]
    else:
        return in_dict

def get_position(metadata):
    try:
        gantry_meta = metadata['lemnatec_measurement_metadata']['gantry_system_variable_metadata']
        gantry_x = gantry_meta["position x [m]"]
        gantry_y = gantry_meta["position y [m]"]
        gantry_z = gantry_meta["position z [m]"]

        sensor_fix_meta = metadata['lemnatec_measurement_metadata']['sensor_fixed_metadata']
        camera_x = sensor_fix_meta['scanner west location in camera box x [m]']
        camera_z = sensor_fix_meta['scanner west location in camera box z [m]']

    except KeyError as err:
        fail('Metadata file missing key: ' + err.args[0])
        return(0, 0, 0)

    try:
        x = float(gantry_x) + float(camera_x)
        y = float(gantry_y)
        z = float(gantry_z) + float(camera_z)
    except ValueError as err:
        fail('Corrupt positions, ' + err.args[0])
    return (x, y, z)

def fail(reason):
    print >> sys.stderr, reason

if __name__ == "__main__":

    main()


text:
from sklearn.model_selection import LeaveOneOut
from sklearn.neural_network import MLPClassifier
from load_models import load_model
from utils import load_data, load_vecs, results_to_file, merge_wi_dicts
from utils import to_np_array
import numpy as np
from scipy.sparse import csc_matrix
import sys
from random import shuffle
import glob
# unique identifiers for different runs (because of random initialisations)
import datetime

def shuffle_examples(x, y):
    inds = list(range(len(x)))

    shuffle(inds)

    x_shuffled = []
    y_shuffled = []

    for i in inds:
        x_shuffled.append(x[i])
        y_shuffled.append(y[i])

    return np.array(x_shuffled), np.array(y_shuffled), inds

def mlp_classification_loo(x, y, shuffle=True):
    loo = LeaveOneOut()
    loo.get_n_splits(x)

    input = len(x[0])
    print('Input size: ', input)

    # Recommended way of setting the nodes in the hidden layer. It is Recommended
    # to start with one hidden layer.
    hidden_layer1_size = int(round((input + 1) * (2 / 3), 0))
    print('Hidden layer size: ', hidden_layer1_size)

    # default solver is adam, but the doc says for smaller data sets, 'lbfgs' performs better.
    mlp = MLPClassifier(hidden_layer_sizes=(hidden_layer1_size), solver='lbfgs')

    predictions = []

    # For the neural net, the order of examples matters. We shuffle the input:

    if shuffle == True:
        x_new, y_new, inds_shuffled = shuffle_examples(x, y)

    elif shuffle == 'reverse':
        x_new = x[::-1]
        y_new = y[::-1]
    else:
        x_new = x
        y_new = y

    for train_index, test_index in loo.split(x_new):
        # All examples
        x_train, x_test = x_new[train_index], x_new[test_index]
        y_train, y_test = y_new[train_index], y_new[test_index]

        mlp.fit(x_train, y_train)
        prediction = mlp.predict(x_test)[0]

        predictions.append(prediction)

    # Map predictions back to original order of examples:

    if shuffle == True:
        predictions_original_order = []

        mapping = []
        for n, i in enumerate(inds_shuffled):
            mapping.append((i, predictions[n]))

        predictions_original_order = [p for i, p in sorted(mapping)]

        for p, po, i, n in zip(predictions, predictions_original_order, inds_shuffled, range(len(inds_shuffled))):

            # check:

            if predictions_original_order[i] != p:
                print('something went wrong: ', n, po, i, p)

        # additional check:
        # mapp examples back and see if the matrices are the same

    elif shuffle == 'reverse':
        predictions_original_order = predictions[::-1]
    else:
        predictions_original_order = predictions

    return predictions_original_order

def mlp_classification(x_train, y_train, x_test, shuffle=True):
    input = len(x_train[0])
    print('Input size: ', input)

    # Recommended way of setting the nodes in the hidden layer. It is Recommended
    # to start with one hidden layer.
    hidden_layer1_size = int(round((input + 1) * (2 / 3), 0))
    print('Hidden layer size: ', hidden_layer1_size)

    # default solver is adam, but the doc says for smaller data sets, 'lbfgs' performs better.
    mlp = MLPClassifier(hidden_layer_sizes=(hidden_layer1_size), solver='lbfgs')

    predictions = []

    # For the neural net, the order of examples matters. We shuffle the input:

    if shuffle == True:
        x_train_new, y_train_new, inds_shuffled = shuffle_examples(x_train, y_train)

    elif shuffle == 'reverse':
        x_train_new = x_train[::-1]
        y_train_new = y_train[::-1]
    else:
        x_train_new = x_train
        y_train_new = y_train

    mlp.fit(x_train_new, y_train_new)
    predictions = mlp.predict(x_test)

    return predictions

def neural_net_classification_loo(model, feature, shuffle=True):
    final_predictions = []

    words_pos, words_neg = load_data(feature)

    vecs_pos, wi_dict_pos = load_vecs(model, words_pos)
    vecs_neg, wi_dict_neg = load_vecs(model, words_neg)

    words = words_pos + words_neg
    x = vecs_pos + vecs_neg

    # Transform sparse vectors to np vectors

    if type(x[0]) != np.ndarray:
        x_list = []

        for vec in x:
            x_list.append(vec.toarray()[0])

        x = np.array(x_list)

    else:
        x = np.array(x)

    y = [1 for vec in vecs_pos]
    [y.append(0) for vec in vecs_neg]

    y = np.array(y)

    wi_dict = merge_wi_dicts(wi_dict_pos, wi_dict_neg)

    predictions = mlp_classification_loo(x, y, shuffle=shuffle)

    for word in words:

        vec_index = wi_dict[word]

        if vec_index != 'OOV':

            final_predictions.append(predictions[vec_index])

        else:
            final_predictions.append('OOV')

    return words, final_predictions

def neural_net_classification(model, feature_train, feature_test, shuffle=True):
    final_predictions = []

    words_pos_train, words_neg_train = load_data(feature_train)

    vecs_pos_train, wi_dict_pos_train = load_vecs(model, words_pos_train)
    vecs_neg_train, wi_dict_neg_train = load_vecs(model, words_neg_train)
    wi_dict_train = merge_wi_dicts(wi_dict_pos_train, wi_dict_neg_train)

    words_train = words_pos_train + words_neg_train
    x_train = vecs_pos_train + vecs_neg_train
    y_train = [1 for vec in vecs_pos_train]
    [y_train.append(0) for vec in vecs_neg_train]

    y_train = np.array(y_train)

    words_pos_test, words_neg_test = load_data(feature_test)

    vecs_pos_test, wi_dict_pos_test = load_vecs(model, words_pos_test)
    vecs_neg_test, wi_dict_neg_test = load_vecs(model, words_neg_test)
    wi_dict_test = merge_wi_dicts(wi_dict_pos_test, wi_dict_neg_test)

    words_test = words_pos_test + words_neg_test
    x_test = vecs_pos_test + vecs_neg_test
    # Transform sparse vectors to np vectors

    # transform to np array:

    x_train = to_np_array(x_train)
    x_test = to_np_array(x_test)

    predictions = mlp_classification(x_train, y_train, x_test, shuffle=shuffle)

    for word in words_test:

        vec_index = wi_dict_test[word]

        if vec_index != 'OOV':

            final_predictions.append(predictions[vec_index])

        else:
            final_predictions.append('OOV')

    return words_test, final_predictions

def main():
    experiment_name = 'neural_net_classification'

    path_to_model = sys.argv[1]
    model_name = sys.argv[2]
    model_type = sys.argv[3]
    features = sys.argv[4]
    shuffle = sys.argv[5]
    test = sys.argv[6]

    data = '../data/experiment/'

    print(glob.glob(data + '*-pos.txt'))

    if (features == 'train') and (test == 'test'):
        print('train-test')
        features = sorted([f.split('/')[-1].split('-')[0] for f in glob.glob(data + '*_train-pos.txt')])
        test_features = sorted([f.split('/')[-1].split('-')[0] for f in glob.glob(data + '*_test-pos.txt')])

    elif (features == 'all') and (test == 'loo'):
        features = [f.split('/')[-1].split('-')[0] for f in glob.glob(data + '*-pos.txt') \
                    if (not 'train' in f) and (not 'test' in f)]

    else:
        features = [features]

    model = load_model(path_to_model, model_type)

    if shuffle == 'shuffle':
        shuffle = True
        sh_par = 'shuff'
    elif shuffle == 'ordered':
        sh_par = 'ordered'
    elif shuffle == 'reverse':
        sh_par = 'reverse'

    ts = str(datetime.datetime.now()).replace(' ', '-').replace('/', '-').replace('.', '-')
    # par = 'default-'+sh_par+'-'+ts

    for no, feat in enumerate(features):

        print(feat, no + 1, '/', len(features))

        if test == 'loo':
            par = 'default-' + sh_par + '-' + ts + '-loo'
            words, predictions = neural_net_classification_loo(model, feat, shuffle=shuffle)
            results_to_file(words, predictions, model_name, experiment_name, feat, par)

        else:
            feature_train = feat
            feature_test = test_features[no]
            print(feature_train, feature_test)
            words, predictions = neural_net_classification(model, feature_train, feature_test, shuffle=shuffle)
            par = 'default-' + sh_par + '-' + ts + '-test'
            results_to_file(words, predictions, model_name, experiment_name, feature_test, par)

if __name__ == '__main__':
    main()


text:
import cv2
import numpy as np
from matplotlib import pyplot as plt
from matplotlib.patches import Rectangle
from typing import List, Tuple

from fast_segmentation.core.consts import LABEL_TO_COLOR, IGNORE_LABEL, OTHER_LABEL, PIXELS_SCALE

def labels_mask_to_colored_image(mask: np.ndarray) -> np.ndarray:
    """
    Converts the mask to an image - replaces the labels with the matching colors

    Args:
        mask: a grayscale semantic segmentation annotation mask with values up to the number of classes
              (like in the consts)

    Returns:
        colored annotation mask - rgb
    """
    return np.asarray(list(LABEL_TO_COLOR.values()), dtype=np.uint8)[mask]

def put_colored_annotation_on_image(image: np.ndarray, annotation: np.ndarray, opacity: float = 0.5) -> np.ndarray:
    """
    Creates an image with annotation on it

    Args:
        image: the original image in RGB format
        annotation: colored annotation mask in RGB format
        opacity: the amount of opacity to use in the visualization

    Returns:
        the combination of the image and the annotation
    """
    return (((1 - opacity) * image) + (opacity * annotation)).astype("uint8")

def get_legends(colors: List[Tuple[int]]) -> List[Rectangle]:
    """
    Creates colored rects for legends of image

    Args:
        colors: a list of colors - a color is a tuple of rgb

    Returns:
        a list of colored rectangles
    """
    legends = [Rectangle((0, 0), 1, 1, color=np.asarray(color[::-1]) / PIXELS_SCALE) for color in colors]

    return legends

def save_image_with_legends_and_labels(save_path: str, image: np.ndarray, legends: List[Rectangle],
                                       labels: List[str]) -> (plt.Figure, plt.Axes):
    """
    Saves the given image with the given legends rects and labels

    Args:
        save_path: the path to the folder to save the image
        image: the image to put the legends on
        legends: a list of colored rects
        labels: a list of the names of the legends

    Returns:
        the figure and the axes of matplotlib
    """
    figure, axes = plt.subplots()

    axes.imshow(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
    axes.legend(legends, labels)
    axes.axis('off')

    axes.plot()
    plt.savefig(save_path, bbox_inches='tight', pad_inches=0)

    return figure, axes

def save_labels_mask_with_legend(mask: np.ndarray, save_path: str) -> (plt.Figure, plt.Axes):
    """
    Converts the mask to a colored rgb mask and saves the new mask to the given path

    Args:
        mask: a grayscale mask with values up to the number of classes (like in the consts)
        save_path: the path to save the new mask

    Returns:
        the figure and the axes of matplotlib
    """
    mask[mask == IGNORE_LABEL] = OTHER_LABEL
    image = labels_mask_to_colored_image(mask)
    labels = list(LABEL_TO_COLOR.keys())
    legends = get_legends(list(LABEL_TO_COLOR.values()))

    figure, axes = save_image_with_legends_and_labels(save_path, image, legends, labels)

    return figure, axes

max_stars_repo_name: eilonshi/tevel-segmentation

max_issues_repo_name: eilonshi/tevel-segmentation

max_forks_repo_name: eilonshi/tevel-segmentation


text:
# Beispielprogramm für das Buch "Python Challenge"
#
# Copyright 2020 by Michael Inden

import numpy as np

from ch06_arrays.intro.intro_directions_example import Direction

def main():
    world = np.array([list("ABCDEF"),
                      list("GHIJKL"),
                      list("MNOPQR"),
                      list("abcdef"),
                      list("ghijkl")])

    dir = Direction.provide_random_direction()
    print("Direction:", dir.name)

    pos_x = 0
    pos_y = 0
    steps = 0

    while steps < 25:
        print(world[pos_y][pos_x], " ", end="")

        dx, dy = dir.to_dx_dy()
        if not is_on_board(world, pos_x + dx, pos_y + dy):
            dir = select_new_dir(world, dir, pos_x, pos_y)
            dx, dy = dir.to_dx_dy()
            print("\nNew Direction:", dir.name)

        pos_x += dx
        pos_y += dy
        steps += 1

def select_new_dir(world, dir, pos_x, pos_y):
    old_dir = dir
    while True:
        dir = Direction.provide_random_direction()
        dx, dy = dir.to_dx_dy()
        if old_dir != dir and is_on_board(world, pos_x + dx, pos_y + dy):
            break

    return dir

def is_on_board(values, next_pos_x, next_pos_y):
    max_y, max_x = values.shape

    return 0 <= next_pos_x < max_x and 0 <= next_pos_y < max_y

if __name__ == "__main__":
    main()

max_stars_repo_name: Kreijeck/learning

max_issues_repo_name: Kreijeck/learning

max_forks_repo_name: Kreijeck/learning


text:
# Copyright 2020 AstroLab Software
# Author: Julien Peloton
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pyspark.sql.functions import pandas_udf, PandasUDFType
from pyspark.sql.types import IntegerType

import pandas as pd
import numpy as np

import os

from fink_science import __file__

from fink_science.tester import spark_unit_tests

@pandas_udf(IntegerType(), PandasUDFType.SCALAR)
def nalerthist(magpsf) -> pd.Series:
    """ Compute the number of detections contained in the alert (current+history)
    Upper limits are not counted.

    Parameters
    ----------
    magpsf: Spark DataFrame Columns
        Magnitude from PSF-fit photometry

    Returns
    ----------
    probabilities: 1D np.array of int
        Number of detections contained in the alert (current+history)
        Upper limits are not counted.

    Examples
    ----------
    >>> from fink_science.utilities import concat_col
    >>> from pyspark.sql import functions as F

    >>> df = spark.read.load(ztf_alert_sample)

    # Append temp columns with historical + current measurements
    >>> df = concat_col(df, 'magpsf', prefix='c')

    # Perform the fit + classification (default model)
    >>> df = df.withColumn('nalerthist', nalerthist(df['cmagpsf']))

    >>> df.agg({"nalerthist": "min"}).collect()[0][0]
    1

    >>> df.agg({"nalerthist": "max"}).collect()[0][0]
    9
    """
    nalerthist = magpsf.apply(lambda x: np.sum(np.array(x) == np.array(x)))
    return nalerthist

if __name__ == "__main__":
    """ Execute the test suite """

    globs = globals()
    path = os.path.dirname(__file__)

    ztf_alert_sample = 'file://{}/data/alerts/alerts.parquet'.format(path)
    globs["ztf_alert_sample"] = ztf_alert_sample

    # Run the test suite
    spark_unit_tests(globs)

max_stars_repo_name: tallamjr/fink-science

max_issues_repo_name: tallamjr/fink-science

max_forks_repo_name: tallamjr/fink-science


text:
import os
import sys
import time

sys.path.append(os.getcwd())

import numpy
import torch
import wx
import PIL.Image
import pyvirtualcam
import cv2
import dlib
import face_recognition_models

from tha2.poser.poser import Poser
from tha2.mocap.face_landmarks_converter import FaceLandmarksConverter
from tha2.util import extract_PIL_image_from_filelike, resize_PIL_image, extract_pytorch_image_from_PIL_image, convert_output_image_from_torch_to_numpy

face_detector = dlib.get_frontal_face_detector()
pose_predictor = dlib.shape_predictor(face_recognition_models.pose_predictor_model_location())

class MainFrame(wx.Frame):
    def __init__(self, poser: Poser, pose_converter: FaceLandmarksConverter, device: torch.device):
        super().__init__(None, wx.ID_ANY, "Webcam Face Tracking")
        self.pose_converter = pose_converter
        self.poser = poser
        self.device = device

        self.wx_source_image = None
        self.torch_source_image = None
        self.source_image_string = "Nothing yet!"

        self.main_sizer = wx.BoxSizer(wx.HORIZONTAL)
        self.SetSizer(self.main_sizer)
        self.SetAutoLayout(1)
        self.input_panel = wx.Panel(self, size=(256, 312), style=wx.SIMPLE_BORDER)
        self.input_panel_sizer = wx.BoxSizer(wx.VERTICAL)
        self.input_panel.SetSizer(self.input_panel_sizer)
        self.input_panel.SetAutoLayout(1)
        self.main_sizer.Add(self.input_panel, 0, wx.FIXED_MINSIZE)

        self.source_image_panel = wx.Panel(self.input_panel, size=(256, 256), style=wx.SIMPLE_BORDER)
        self.source_image_panel.Bind(wx.EVT_PAINT, self.paint_source_image_panel)
        self.input_panel_sizer.Add(self.source_image_panel, 0, wx.FIXED_MINSIZE)

        self.load_image_button = wx.Button(self.input_panel, wx.ID_ANY, "Load Image")
        self.input_panel_sizer.Add(self.load_image_button, 1, wx.EXPAND)
        self.load_image_button.Bind(wx.EVT_BUTTON, self.load_image)

        self.input_panel_sizer.Fit(self.input_panel)

        self.pose_converter.init_pose_converter_panel(self)

        self.right_panel = wx.Panel(self, style=wx.SIMPLE_BORDER)
        right_panel_sizer = wx.BoxSizer(wx.VERTICAL)
        self.right_panel.SetSizer(right_panel_sizer)
        self.right_panel.SetAutoLayout(1)
        self.main_sizer.Add(self.right_panel, 0, wx.EXPAND)

        self.result_image_panel = wx.Panel(self.right_panel, size=(256, 256), style=wx.SIMPLE_BORDER)
        self.result_image_panel.Bind(wx.EVT_PAINT, self.paint_result_image_panel)
        self.output_index_choice = wx.Choice(
            self.right_panel,
            choices=[str(i) for i in range(self.poser.get_output_length())])
        self.output_index_choice.SetSelection(0)
        right_panel_sizer.Add(self.result_image_panel, 0, wx.FIXED_MINSIZE)
        right_panel_sizer.Add(self.output_index_choice, 0, wx.EXPAND)

        self.fps_text = wx.StaticText(self.right_panel, label="")
        right_panel_sizer.Add(self.fps_text, wx.SizerFlags().Border())
        right_panel_sizer.Fit(self.right_panel)

        self.main_sizer.Fit(self)

        self.Bind(wx.EVT_CLOSE, self.on_close)

        self.timer = wx.Timer(self, wx.ID_ANY)
        self.Bind(wx.EVT_TIMER, self.update_result_image_panel, self.timer)
        self.timer.Start(30)

        self.last_pose = None
        self.last_output_index = self.output_index_choice.GetSelection()
        self.last_output_numpy_image = None
        self.camera = pyvirtualcam.Camera(256, 256, 60)
        self.capture = cv2.VideoCapture(0, cv2.CAP_DSHOW)

    def on_close(self, event: wx.Event):
        self.timer.Stop()
        self.capture.release()
        cv2.destroyAllWindows()
        event.Skip()

    def load_image(self, event: wx.Event):
        dir_name = "data/illust"
        file_dialog = wx.FileDialog(self, "Choose an image", dir_name, "", "*.png", wx.FD_OPEN)
        if file_dialog.ShowModal() == wx.ID_OK:
            image_file_name = os.path.join(file_dialog.GetDirectory(), file_dialog.GetFilename())
            try:
                pil_image = resize_PIL_image(extract_PIL_image_from_filelike(image_file_name))
                w, h = pil_image.size
                self.wx_source_image = wx.Bitmap.FromBufferRGBA(w, h, pil_image.convert("RGBA").tobytes())
                self.torch_source_image = extract_pytorch_image_from_PIL_image(pil_image).to(self.device)

                self.Refresh()
            except:
                message_dialog = wx.MessageDialog(self, "Could not load image " + image_file_name, "Poser", wx.OK)
                message_dialog.ShowModal()
                message_dialog.Destroy()
        file_dialog.Destroy()

    def paint_source_image_panel(self, event: wx.Event):
        if self.wx_source_image is None:
            self.draw_source_image_string(self.source_image_panel, use_paint_dc=True)
        else:
            dc = wx.PaintDC(self.source_image_panel)
            dc.Clear()
            dc.DrawBitmap(self.wx_source_image, 0, 0, True)

    def paint_result_image_panel(self, event: wx.Event):
        self.last_pose = None

    def draw_source_image_string(self, widget, use_paint_dc: bool = True):
        if use_paint_dc:
            dc = wx.PaintDC(widget)
        else:
            dc = wx.ClientDC(widget)
        dc.Clear()
        font = wx.Font(wx.FontInfo(14).Family(wx.FONTFAMILY_SWISS))
        dc.SetFont(font)
        w, h = dc.GetTextExtent(self.source_image_string)
        dc.DrawText(self.source_image_string, 128 - w // 2, 128 - h // 2)

    def update_result_image_panel(self, event: wx.Event):
        tic = time.perf_counter()

        if self.torch_source_image is None:
            current_pose = None
        else:
            ret, frame = self.capture.read()
            locations = face_detector(frame, 1)
            if len(locations) == 0:
                return
            landmarks = [[p.x, p.y] for p in pose_predictor(frame, locations[0]).parts()]
            current_pose = self.pose_converter.convert(landmarks)

        if self.last_pose is not None \
                and self.last_pose == current_pose \
                and self.last_output_index == self.output_index_choice.GetSelection():
            return
        self.last_pose = current_pose
        self.last_output_index = self.output_index_choice.GetSelection()

        if self.torch_source_image is None:
            self.draw_source_image_string(self.result_image_panel, use_paint_dc=False)
            return

        pose = torch.tensor(current_pose, dtype=torch.float, device=self.device)
        output_index = self.output_index_choice.GetSelection()
        output_image = self.poser.pose(self.torch_source_image, pose, output_index)[0].detach().cpu()
        numpy_image = convert_output_image_from_torch_to_numpy(output_image)
        self.last_output_numpy_image = numpy_image
        wx_image = wx.ImageFromBuffer(
            numpy_image.shape[0],
            numpy_image.shape[1],
            numpy_image[:, :, 0:3].tobytes(),
            numpy_image[:, :, 3].tobytes())
        wx_bitmap = wx_image.ConvertToBitmap()

        dc = wx.ClientDC(self.result_image_panel)
        dc.Clear()
        dc.DrawBitmap(wx_bitmap, (256 - numpy_image.shape[0]) // 2, (256 - numpy_image.shape[1]) // 2, True)
        self.camera.send(numpy_image)

        toc = time.perf_counter()
        elapsed_time = toc - tic
        fps = 1.0 / elapsed_time
        self.fps_text.SetLabelText("FPS = %0.2f" % fps)

if __name__ == "__main__":
    import tha2.poser.modes.face_landmarks
    import tha2.poser.modes.mode_20

    cuda = torch.device('cuda')
    poser = tha2.poser.modes.mode_20.create_poser(cuda)
    pose_converter = tha2.poser.modes.face_landmarks.create_face_landmarks_converter()

    app = wx.App()
    main_frame = MainFrame(poser, pose_converter, cuda)
    main_frame.Show(True)
    app.MainLoop()


text:
import numpy as np
from sklearn.utils import resample

def get_datapoint_ids(label, list):
    """
    Returns the data points for the given labels.
    :param label: []
    :param list: []
    """
    return np.array(np.where(list == label)).ravel()

def generate_new_labels(challenge_ids, labels, p, count):
    """
    Randomly upsamples challenge_ids and associates labels with data points using probabilities p.
    :param challenge_ids: [int]
    :param labels: []
    :param p: [float]
    :param count: int
    :return: 2D ndarray of challenge_ids, label
    """
    upsampled = resample(challenge_ids, n_samples=count)
    new_labels = np.random.choice(labels, count, True, p)
    return np.array([upsampled, new_labels]).transpose()

def generate_new_annotations_confusionmatrix(cm, classes, datapoints, count=1000, normalize=False):
    """
    Generates annotations for each class given by the confusion matrix cm using generate new labels.
    :param cm: 2D array confusion matrix
    :param classes: []
    :param datapoints: []
    :param count: int
    :param normalize: bool
    :return: 2d ndarray of annotations
    """
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")

    annotations = np.ndarray(shape=(0, 2))
    for i in range(len(classes)):
        challengeIDs = get_datapoint_ids(classes[i], datapoints)
        newann = generate_new_labels(challengeIDs, classes, cm[i], count)
        annotations = np.append(annotations, newann, axis=0)

    return annotations

max_stars_repo_name: transimprove/Transimprove

max_issues_repo_name: transimprove/Transimprove

max_forks_repo_name: transimprove/Transimprove