text:
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import torch
import torchdyn; from torchdyn.models import *;

import matplotlib.pyplot as plt
from matplotlib import cm
import numpy as np
from matplotlib.colors import Normalize
from scipy.interpolate import interpn

def grid_apply(X, Y, F):
    z = torch.cat([X.reshape(-1, 1), Y.reshape(-1, 1)], 1)
    f = F(z).detach().cpu() ; U, V = f[:,0].reshape(X.shape), f[:, 1].reshape(X.shape)
    return U, V

def plot(model, prior, target, step, save=True, show=False):
    model = model.cpu()
    sample = prior.sample(torch.Size([1 << 10])).cpu()
    traj = model[1].trajectory(Augmenter(1, 1)(sample), s_span=torch.linspace(0, 1, 100)).detach().cpu() ;
    traj = traj[:,:,1:]
    xx = torch.linspace(-2, 2, 100) ; X, Y = torch.meshgrid(xx, xx)
    U, V = grid_apply(X, Y, model[1].defunc.m.net)
    #
    fig, ax = plt.subplots(1, 2, figsize=(10, 5))
    ax[0].scatter(sample[:,0], sample[:,1], s=10, alpha=0.3, c='black')
    for i in range(1<<10):
        ax[0].plot(traj[:,i,0], traj[:,i,1], alpha=0.2, c='olive')
    ax[0].scatter(target[:,0].cpu(), target[:,1].cpu(), s=4, alpha=.8, c='orange')
    ax[0].scatter(traj[-1,:,0], traj[-1,:,1], s=4, alpha=.8, c='blue')
    ax[1].contourf(X, Y, torch.sqrt(U**2+V**2), 100, cmap='inferno')
    ax[1].streamplot(xx.numpy(), xx.numpy(), U.T, V.T, color='w', density=2)
    ax[0].set_xlim([-2, 2]) ; ax[0].set_ylim([-2, 2]) ; ax[1].set_xlim([-2, 2]) ; ax[1].set_ylim([-2, 2])
    ax[0].set_title('trajectories / reconstructed density') ; ax[1].set_title('learned vector field')
    if save: plt.savefig('train_plots/ffjord_train_%.3d' % step, dpi=100)
    if not show: plt.close(fig)

def density_scatter(x, y, ax=None, bins=20, **kwargs):
    x, y = x.numpy(), y.numpy()
    data , x_e, y_e = np.histogram2d(x, y, bins = bins, density = True )
    z = interpn((0.5*(x_e[1:] + x_e[:-1]), 0.5*(y_e[1:]+y_e[:-1])), data, np.vstack([x,y]).T , method="splinef2d", bounds_error=False)
    z[np.where(np.isnan(z))] = 0.0
    cmap = cm.inferno
    ax.scatter( x, y, c=z, **kwargs , s=1, cmap=cmap, rasterized=True)
    ax.set_facecolor(cmap(0.))
    norm = Normalize(vmin = np.min(z), vmax = np.max(z))
    return ax

def plot_ffjord(model, hsolver, prior, step, save=True, show=False):
    model = model.cpu() ; hsolver = hsolver.cpu()
    x = prior.sample(torch.Size([1 << 10]))
    logpx = prior.log_prob(x)[:,None].to(x)
    xtrJ = torch.cat([logpx, x], 1).cpu()
    s_span = torch.linspace(0, 1, 2)
    dopri_traj = model[1].trajectory(xtrJ, s_span).detach()
    solver_traj = hsolver.trajectory(xtrJ.detach(), s_span).detach()
    heun_traj = hsolver.trajectory(xtrJ.detach(), s_span, nn=0).detach()
    # plot
    fig, ax = plt.subplots(1, 3, figsize=(13.4, 4))
    density_scatter(dopri_traj[-1,:, 1], dopri_traj[-1,:,2], ax[0])
    density_scatter(solver_traj[-1,:,1], solver_traj[-1,:,2], ax[1])
    density_scatter(heun_traj[-1,:,1], heun_traj[-1,:,2], ax[2])
    ax[0].set_xlim([-2, 2]) ; ax[0].set_ylim([-2, 2]) ; ax[1].set_xlim([-2, 2]) ; ax[1].set_ylim([-2, 2]) ; ax[2].set_xlim([-2, 2]) ; ax[2].set_ylim([-2, 2])
    ax[0].set_title('dopri5') ; ax[1].set_title('HyperHeun') ; ax[2].set_title('Heun')
    if save: plt.savefig('train_plots/hsolver_ffjord_%.3d' % step, dpi=100)
    if not show: plt.close(fig)

max_stars_repo_name: Juju-botu/diffeqml-research

max_issues_repo_name: Juju-botu/diffeqml-research

max_forks_repo_name: Juju-botu/diffeqml-research


text:
import numpy as np
import tensorflow as tf
import hdf5storage
import glob
import TrainNetwork.BaseFunctions as bf
import pickle
import timeit

model = tf.keras.models.load_model('C:/Users/yifan/Google Drive/PythonSync/MovingObjDetector-WAMI.python/Models/BinaryClassification/saved_model.model')
reader_fid = open("C:/Users/yifan/Google Drive/PythonSync/MovingObjDetector-WAMI.python/Models/BinaryClassification/saved_image_norm.model", "rb")
averageX = pickle.load(reader_fid)
reader_fid.close()

traindata_path0 = "E:\WPAFB-data-for-training\Vehicle-Binary-Classifier\winsize10"
traindata_pathset = glob.glob(traindata_path0+"\*.mat")
positive_data_all = []
negative_data_all = []
set_cnt = 0
for idx, thispath in enumerate(traindata_pathset):
    if idx > 300:
        mat = hdf5storage.loadmat(thispath)
        positive_data_tmp = mat.get("positive_dataset")
        negative_data_tmp = mat.get("negative_dataset")
        positive_data_all.extend(positive_data_tmp)
        negative_data_all.extend(negative_data_tmp)
        set_cnt += 1
        print("Read dataset: " + str(set_cnt))

del traindata_path0, traindata_pathset, positive_data_tmp, negative_data_tmp, mat, set_cnt

numPositive = len(positive_data_all)
numNegative = len(negative_data_all)
X_test = np.ndarray((numPositive + numNegative, 4, 21, 21), dtype=np.float32)
Y_test = np.zeros((numPositive + numNegative, 1), dtype=np.uint8)

for i in range(numPositive):
    data_t = np.reshape(positive_data_all[i][:, 0], (1, 21, 21))
    data_tminus1 = np.reshape(positive_data_all[i][:, 1], (1, 21, 21))
    data_tminus2 = np.reshape(positive_data_all[i][:, 2], (1, 21, 21))
    data_tminus3 = np.reshape(positive_data_all[i][:, 3], (1, 21, 21))
    X_tmp = np.concatenate((data_t, data_tminus1, data_tminus2, data_tminus3), axis=0)
    X_test[i] = X_tmp
    Y_test[i] = 0

for i in range(numNegative):
    data_t = np.reshape(negative_data_all[i][:, 0], (1, 21, 21))
    data_tminus1 = np.reshape(negative_data_all[i][:, 1], (1, 21, 21))
    data_tminus2 = np.reshape(negative_data_all[i][:, 2], (1, 21, 21))
    data_tminus3 = np.reshape(negative_data_all[i][:, 3], (1, 21, 21))
    X_tmp = np.concatenate((data_t, data_tminus1, data_tminus2, data_tminus3), axis=0)
    X_test[i+numPositive] = X_tmp
    Y_test[i+numPositive] = 1
print("Generating testing data finished...")

del positive_data_all, negative_data_all, data_t, data_tminus1, data_tminus2, data_tminus3,\
    X_tmp

X_test, _ = bf.DataNormalisationZeroCentred(X_test, averageX)
Y_test = tf.keras.utils.to_categorical(Y_test, 2)
#TestLoss = model.evaluate(X_test, Y_test, batch_size=10000, verbose=1)
starttime = timeit.default_timer()
predictResults = model.predict(X_test, batch_size=5000, verbose=1)
endtime = timeit.default_timer()
print("Processing Time (CNN prediction): " + str(endtime - starttime) + " s... ")

AccurateCnt = 0
for idx in range(len(predictResults)):
    thisResult = np.round(predictResults[idx])
    thisLabel = Y_test[idx]
    if all(thisResult == thisLabel):
        AccurateCnt += 1
therate = AccurateCnt/len(predictResults)
print("Accuracy" + str(therate))

ext: py

reason: import numpy


text:
import tensorflow as tf
import numpy as np

class LimitedEntNormal(tf.distributions.Normal):
    def _entropy(self):
        limit = 2.
        lo, hi = (-limit - self._loc) / self._scale / np.sqrt(2), (limit - self._loc) / self._scale / np.sqrt(2)
        return 0.5 * (self._scale.log() + np.log(2 * np.pi) / 2) * (hi.erf() - lo.erf()) + 0.5 * \
            (tf.exp(-hi * hi) * hi - tf.exp(-lo * lo) * lo)

max_stars_repo_name: LinZichuan/AdMRL

max_issues_repo_name: LinZichuan/AdMRL

max_forks_repo_name: LinZichuan/AdMRL


text:
import pylab
import os
import re
import argparse
import numpy as np

from glob import glob

if __name__=="__main__":

    parser = argparse.ArgumentParser(description="Arguments for plotting losses")

    parser.add_argument("--dir", type=str, \
            help="directory containing logs", default="./log/")

    args = parser.parse_args()

    files = sorted(glob(os.path.join(args.dir, '*.log')))

    for f in files:

        print(f)
        name = os.path.basename(f)[:-4]#f.split('/')[-1]
        train_generator_loss = list()
        train_discriminator_loss = list()

        train_momenta_A2B_loss = list()
        train_momenta_B2A_loss = list()
        train_pitch_A2B_loss = list()
        train_pitch_B2A_loss = list()
        train_mfc_A2B_loss = list()
        train_mfc_B2A_loss = list()

        r = open(f,'r')
        for line in r:
            try:
                if "Train Generator" in line:
                    n = re.findall(r"[-+]?\d*\.\d+e-\d+|[-+]?\d*\.\d+|\d+", line)
                    train_generator_loss.append(-0.8*float(n[0]))
                if "Train Discriminator" in line:
                    n = re.findall(r"[-+]?\d*\.\d+e-\d+|[-+]?\d*\.\d+|\d+", line)
                    train_discriminator_loss.append(-0.9*float(n[0]))
                if "Train Momenta A2B" in line:
                    n = re.findall(r"[-+]?\d*\.\d+e-\d+|[-+]?\d*\.\d+|\d+", line)
                    train_momenta_A2B_loss.append(float(n[1]))
                if "Train Momenta B2A" in line:
                    n = re.findall(r"[-+]?\d*\.\d+e-\d+|[-+]?\d*\.\d+|\d+", line)
                    train_momenta_B2A_loss.append(float(n[1]))
                if "Train Pitch A2B" in line:
                    n = re.findall(r"[-+]?\d*\.\d+e-\d+|[-+]?\d*\.\d+|\d+", line)
                    train_pitch_A2B_loss.append(float(n[1]))
                if "Train Pitch B2A" in line:
                    n = re.findall(r"[-+]?\d*\.\d+e-\d+|[-+]?\d*\.\d+|\d+", line)
                    train_pitch_B2A_loss.append(float(n[1]))
                if "Train Mfc A2B" in line:
                    n = re.findall(r"[-+]?\d*\.\d+e-\d+|[-+]?\d*\.\d+|\d+", line)
                    train_mfc_A2B_loss.append(float(n[1]))
                if "Train Mfc B2A" in line:
                    n = re.findall(r"[-+]?\d*\.\d+e-\d+|[-+]?\d*\.\d+|\d+", line)
                    train_mfc_B2A_loss.append(float(n[1]))

            except Exception as e:
                print(e)

        diff = np.abs(np.asarray(train_generator_loss) - np.asarray(train_discriminator_loss))
        r.close()
        if len(train_generator_loss)>0:
            pylab.figure(figsize=(12,12))
            pylab.plot(train_generator_loss, 'r', label="generator loss", linewidth=3)
            pylab.plot(train_discriminator_loss, 'g', label="discriminator loss", linewidth=3)
            pylab.grid(), pylab.legend(loc=1, fontsize=15)
            pylab.suptitle(name)
            pylab.savefig(args.dir+name+'.png')
            pylab.close()
        elif len(train_momenta_A2B_loss)>0:
            pylab.figure(figsize=(12,12))
            pylab.subplot(131)
            pylab.plot(train_momenta_A2B_loss, 'r', label="Momenta A2B loss")
            pylab.plot(train_momenta_B2A_loss, 'g', label="Momenta B2A loss")
            pylab.grid(), pylab.legend(loc=1)
            pylab.subplot(132)
            pylab.plot(train_pitch_A2B_loss, 'r', label="Pitch A2B loss")
            pylab.plot(train_pitch_B2A_loss, 'g', label="Pitch B2A loss")
            pylab.grid(), pylab.legend(loc=1)
            pylab.subplot(133)
            pylab.plot(train_mfc_A2B_loss, 'r', label="Mfc A2B loss")
            pylab.plot(train_mfc_B2A_loss, 'g', label="Mfc B2A loss")
            pylab.grid(), pylab.legend(loc=1)
            pylab.suptitle(name)
            pylab.savefig(args.dir+name+'.png')
            pylab.close()


text:
import crocoddyl
import numpy as np

class SolverSquashFDDP():
    def __init__(self, problem, squashingModel):
        # General objects
        self.problem = problem
        self.actuation = self.problem.terminalModel.differential.actuation
        self.state = self.problem.terminalModel.state

        # Squashing
        self.squashingModel = squashingModel  # it could also be retrieved from the actuation model
        self.squashingSmoothInit = 0.1
        self.squashingSmoothMult = 0.5
        self.squashingSmooth = self.squashingSmoothInit
        self.squashingLb = self.squashingModel.s_lb
        self.squashingUb = self.squashingModel.s_ub

        # Barrier
        self.barrierQuadraticWeights = 1.0 / np.power(self.squashingSmooth * (self.squashingUb - self.squashingLb), 2)
        self.barrierWeight = 1e-3
        self.barrierInit()

        # SBFFDP parameters
        self.convergenceInit = 1e-2
        self.convergenceStop = 1e-3
        self.convergenceMult = 1e-1
        self.convergence = self.convergenceInit
        self.maxIters = 100
        self.regInit = 1e-9

        # Inner solvers
        self.solverFDDP = crocoddyl.SolverFDDP(self.problem)
        self.solverFDDP.th_stop_gaps = 1e0
        self.solverFDDP.setStoppingCriteria(crocoddyl.StoppingType.StopCriteriaCostReduction)
        self.solverFDDP.setStoppingTest(crocoddyl.StoppingTestType.StopTestGaps)
        self.solverDDP = crocoddyl.SolverDDP(self.problem)
        self.solverDDP.setStoppingCriteria(crocoddyl.StoppingType.StopCriteriaCostReduction)

    def barrierInit(self):
        barrierActivationBounds = crocoddyl.ActivationBounds(self.squashingModel.s_lb, self.squashingModel.s_ub, 1.0)
        barrierActivation = crocoddyl.ActivationModelWeightedQuadraticBarrier(barrierActivationBounds,
                                                                              self.barrierQuadraticWeights)
        # barrierActivation = crocoddyl.ActivationModelQuadraticBarrier(barrierActivationBounds)
        squashingBarrierCost = crocoddyl.CostModelControl(self.state, barrierActivation, self.actuation.nu)

        # we only need to change one running model to change them all

        for idx, m in enumerate(self.problem.runningModels):
            if "sBarrier" not in m.differential.costs.costs:
                m.differential.costs.addCost("sBarrier", squashingBarrierCost, self.barrierWeight)
            self.problem.updateModel(idx, m)

    def setCallbacks(self, callbacks):
        self.solverFDDP.setCallbacks(callbacks)
        self.solverDDP.setCallbacks(callbacks)

    def solve(self, xsInit=[], ssInit=[], maxIter=100, regInit=1e-9):
        self.xs = xsInit
        self.ss = ssInit

        while (self.convergence >= self.convergenceStop):
            self.squashingUpdate()
            self.barrierUpdate()

            self.solverFDDP.th_stop = self.convergence
            self.solverFDDP.solve(self.xs, self.ss, self.maxIters, False, self.regInit)
            self.xs = self.solverFDDP.xs
            self.ss = self.solverFDDP.us

            self.squashingSmooth *= self.squashingSmoothMult
            self.convergence *= self.convergenceMult

        if not self.solverFDDP.isFeasible:
            self.solverDDP.th_stop = self.solverFDDP.th_stop
            self.solverDDP.solve(self.xs, self.ss, self.maxIters, False, self.regInit)
            self.xs = self.solverDDP.xs
            self.ss = self.solverDDP.us

    def squashingUpdate(self):
        self.actuation.squashing.smooth = self.squashingSmooth

    def barrierUpdate(self):
        self.barrierQuadraticWeights = 1.0 / np.power(self.squashingSmooth * (self.squashingUb - self.squashingLb), 2)
        for m in self.problem.runningModels:
            m.differential.costs.costs['sBarrier'].cost.activation.weights = self.barrierQuadraticWeights
            m.differential.costs.costs['sBarrier'].weight = self.barrierWeight
        # self.problem.runningModels[0].differential.costs.costs[
        #     'sBarrier'].cost.activation.weights = self.barrierQuadraticWeights
        # self.problem.runningModels[0].differential.costs.costs['sBarrier'].weight = self.barrierWeight

    def setBarrierWeight(self, weight):
        self.barrierWeight = weight
        # for m in self.problem.runningModels:
        #     m.differential.costs.costs['sBarrier'].weight = self.barrierWeight
        self.problem.runningModels[0].differential.costs.costs['sBarrier'].weight = self.barrierWeight

max_stars_repo_name: PepMS/eagle-mpc

max_issues_repo_name: PepMS/multicopter-mpc

max_forks_repo_name: PepMS/eagle-mpc


text:
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import sys
import os
import time
import numpy as np
from Net import MyModel, LabelSmoothLoss
from sklearn import metrics
from data_loader import my_data_loader
from config import config
from utils import AverageMeter

def get_hier_rel(hier_rel, scope):
    new_rel = []
    index = 0
    for s in scope:
        new_rel += [hier_rel[index]] * (s[1]-s[0])
        index += 1
    return torch.stack(new_rel)

def train(train_loader, test_loader, opt):
    if opt['model'] == 'GHE-LPC':
        model = MyModel(
            train_loader.dataset.vec_save_dir,
            train_loader.dataset.hier1_rel_num(),
            train_loader.dataset.hier2_rel_num(),
            train_loader.dataset.hier3_rel_num(),
            lambda_embed=opt['lambda_embed'],
            use_ghe=opt['use_ghe'],
            hier_encoder_heads=opt['hier_encoder_heads']
            )

        hier1_criterion_S = LabelSmoothLoss(smoothing=0.1)
        hier2_criterion_S = LabelSmoothLoss(smoothing=0.1)
        hier3_criterion_S = LabelSmoothLoss(smoothing=0.1)
        criterion = nn.CrossEntropyLoss()

    if torch.cuda.is_available():
        model = model.cuda()

    print(model)
    if opt['use_ghe']:
        print("Use Hierarchy Encoder to generate GHE!!")
    if opt['use_lpc']:
        print("Consider Local Probability Constaints!! alpha = ", opt['lpc_alpha'])

    optimizer = optim.SGD(model.parameters(), lr=opt['lr'], weight_decay=1e-5)
    not_best_count = 0
    best_auc = 0
    if not os.path.exists(opt['save_dir']):
        os.mkdir(opt['save_dir'])
    ckpt = os.path.join(opt['save_dir'], 'model.pth.tar')
    for epoch in range(opt['epoch']):
        model.train()
        print("\n=== Epoch %d train ===" % epoch)
        avg_loss = AverageMeter()
        avg_acc = AverageMeter()
        avg_pos_acc = AverageMeter()
        for i, data in enumerate(train_loader):
            optimizer.zero_grad()
            if torch.cuda.is_available():
                for d in range(len(data)):
                    data[d] = data[d].cuda()
            if opt['model'] == 'GHE-LPC':
                word, pos1, pos2, ent_order, ent_pos, index1, index2, ent1, ent2, mask, length, scope, hier1_rel, hier2_rel, rel = data
                output, output_hier1_S, output_hier2_S, output_hier3_S, \
                    hier1_next_logits, hier2_next_logits, hier2_prev_logits, hier3_prev_logits = \
                        model(word, pos1, pos2, ent_order, ent_pos, index1, index2, ent1, ent2, mask, scope, length)

                hier1_rel = get_hier_rel(hier1_rel, scope)  # 一个bag 一个label
                hier2_rel = get_hier_rel(hier2_rel, scope)
                hier3_rel = get_hier_rel(rel, scope)
                loss_hier1_S = hier1_criterion_S(output_hier1_S, hier1_rel)
                loss_hier2_S = hier2_criterion_S(output_hier2_S, hier2_rel)
                loss_hier3_S = hier3_criterion_S(output_hier3_S, hier3_rel)

                # print(output_hier1_S.shape, hier2_prev_logits.shape, output_hier2_S.shape, hier3_prev_logits.shape)
                loss_label_reg_1 = F.kl_div(F.softmax(output_hier1_S, dim=-1).log(), hier2_prev_logits, reduction='batchmean')
                loss_label_reg_2 = F.kl_div(F.softmax(output_hier2_S, dim=-1).log(), hier3_prev_logits, reduction='batchmean')

            loss = criterion(output, rel)
            loss += loss_hier1_S
            loss += loss_hier2_S
            loss += loss_hier3_S
            if opt['use_lpc']: # and epoch >= 5:
                loss_label_reg = loss_label_reg_1 + loss_label_reg_2
                loss += opt['lpc_alpha'] * loss_label_reg

            _, pred = torch.max(output, -1)
            acc = (pred == rel).sum().item() / rel.shape[0]
            pos_total = (rel != 0).sum().item()
            pos_correct = ((pred == rel) & (rel != 0)).sum().item()
            if pos_total > 0:
                pos_acc = pos_correct / pos_total
            else:
                pos_acc = 0
            # Log
            avg_loss.update(loss.item(), 1)
            avg_acc.update(acc, 1)
            avg_pos_acc.update(pos_acc, 1)
            sys.stdout.write('\rstep: %d | loss: %f, acc: %f, pos_acc: %f'%(i, avg_loss.avg, avg_acc.avg, avg_pos_acc.avg))
            sys.stdout.flush()
            # Optimize
            loss.backward()
            optimizer.step()

        if (epoch + 1) % opt['val_iter'] == 0 and avg_pos_acc.avg > 0.3:
            print("\n=== Epoch %d val ===" % epoch)
            y_true, y_pred = valid(test_loader, model, opt)
            auc = metrics.average_precision_score(y_true, y_pred)
            print("\n[TEST] auc: {}".format(auc))
            if auc > best_auc:
                print("Best result!")
                best_auc = auc
                torch.save({'state_dict': model.state_dict()}, ckpt)
                not_best_count = 0
            else:
                not_best_count += 1
            if not_best_count >= opt['early_stop']:
                break

def valid(test_loader, model, opt):
    model.eval()
    avg_acc = AverageMeter()
    avg_pos_acc = AverageMeter()
    y_true = []
    y_pred = []
    with torch.no_grad():
        for i, data in enumerate(test_loader):
            if torch.cuda.is_available():
                data = [x.cuda() for x in data]
            if opt['model'] == 'GHE-LPC':
                word, pos1, pos2, ent_order, ent_pos, index1, index2, ent1, ent2, mask, length, scope, hier1_rel, hier2_rel, rel = data
                output, output_hier1_S, output_hier2_S, output_hier3_S, \
                    hier1_next_logits, hier2_next_logits, hier2_prev_logits, hier3_prev_logits = \
                        model(word, pos1, pos2, ent_order, ent_pos, index1, index2, ent1, ent2, mask, scope, length)

            output = torch.softmax(output, -1)
            label = rel.argmax(-1)
            _, pred = torch.max(output, -1)
            acc = (pred == label).sum().item() / label.shape[0]
            pos_total = (label != 0).sum().item()
            pos_correct = ((pred == label) & (label != 0)).sum().item()
            if pos_total > 0:
                pos_acc = pos_correct / pos_total
            else:
                pos_acc = 0
            # Log
            avg_acc.update(acc, 1)
            avg_pos_acc.update(pos_acc, 1)
            sys.stdout.write('\rstep: %d | acc: %f, pos_acc: %f'%(i, avg_acc.avg, avg_pos_acc.avg))
            sys.stdout.flush()
            y_true.append(rel[:, 1:])  # exclude NA
            y_pred.append(output[:, 1:])
    y_true = torch.cat(y_true).reshape(-1).detach().cpu().numpy()
    y_pred = torch.cat(y_pred).reshape(-1).detach().cpu().numpy()
    return y_true, y_pred

def test(test_loader, opt):
    print("=== Test ===")
    # Load model
    save_dir = opt['save_dir']
    if opt['model'] == 'GHE-LPC':
        model = MyModel(
            train_loader.dataset.vec_save_dir,
            train_loader.dataset.hier1_rel_num(),
            train_loader.dataset.hier2_rel_num(),
            train_loader.dataset.hier3_rel_num(),
            lambda_embed=opt['lambda_embed'],
            use_ghe=opt['use_ghe'],
            hier_encoder_heads=opt['hier_encoder_heads']
            )
    if torch.cuda.is_available():
        model = model.cuda()
    state_dict = torch.load(os.path.join(save_dir, 'model.pth.tar'))['state_dict']
    own_state = model.state_dict()
    for name, param in state_dict.items():
        if name not in own_state:
            continue
        own_state[name].copy_(param)

    y_true, y_pred = valid(test_loader, model, opt)
    # AUC
    auc = metrics.average_precision_score(y_true, y_pred)
    print("\n[TEST] auc: {}".format(auc))
    # P@N
    order = np.argsort(-y_pred)
    p100 = (y_true[order[:100]]).mean() * 100
    p200 = (y_true[order[:200]]).mean() * 100
    p300 = (y_true[order[:300]]).mean() * 100
    p500 = (y_true[order[:500]]).mean() * 100
    p1000 = (y_true[order[:1000]]).mean() * 100
    p2000 = (y_true[order[:2000]]).mean() * 100
    print("P@100: {0:.1f}, P@200: {1:.1f}, P@300: {2:.1f}, P@500: {3:.1f}, P@1000: {4:.1f}, P@2000: {5:.1f}, Mean: {6:.1f}".format(p100, p200, p300, p500, p1000, p2000, (p100 + p200 + p300 + p500 + p1000 + p2000) / 6))
    # PR
    order = np.argsort(y_pred)[::-1]
    correct = 0.
    total = y_true.sum()
    precision = []
    recall = []
    for i, o in enumerate(order):
        correct += y_true[o]
        precision.append(float(correct) / (i + 1))
        recall.append(float(correct) / total)
    precision = np.array(precision)
    recall = np.array(recall)
    print("Saving result")
    np.save(os.path.join(save_dir, 'precision-' + opt['model'] + '.npy'), precision)
    np.save(os.path.join(save_dir, 'recall-' + opt['model'] + '.npy'), recall)
    return y_true, y_pred

if __name__ == '__main__':
    print("reading parameters: config.py")
    opt = vars(config())
    if opt['model'] == 'GHE-LPC':
        train_loader = my_data_loader(opt['train'], opt, shuffle=True, training=True)
        test_loader = my_data_loader(opt['test'], opt, shuffle=False, training=False)
    train(train_loader, test_loader, opt)
    y_true, y_pred = test(test_loader, opt)

max_stars_repo_name: RidongHan/GHE-LPC

max_issues_repo_name: RidongHan/GHE-LPC

max_forks_repo_name: RidongHan/GHE-LPC


text:
#!/usr/bin/env python3

import glob
import os
import numpy as np
import argparse
import pickle
import logging

from sklearn import model_selection

import model
from util import io_util, logging_util
from data_class import opunit_data
from info import data_info
from training_util import data_transforming_util, result_writing_util

from type import Target

np.set_printoptions(precision=4)
np.set_printoptions(edgeitems=10)
np.set_printoptions(suppress=True)

class MiniTrainer:
    """
    Trainer for the mini models
    """

    def __init__(self, input_path, model_metrics_path, ml_models, test_ratio):
        self.input_path = input_path
        self.model_metrics_path = model_metrics_path
        self.ml_models = ml_models
        self.test_ratio = test_ratio

    def train(self):
        """Train the mini-models

        :return: the map of the trained models
        """

        data_list = []

        # First get the data for all mini runners
        for filename in glob.glob(os.path.join(self.input_path, '*.csv')):
            print(filename)
            data_list += opunit_data.get_mini_runner_data(filename)

        model_map = {}
        # train the models for all the operating units
        for data in data_list:
            x_train, x_test, y_train, y_test = model_selection.train_test_split(data.x, data.y,
                                                                                test_size=self.test_ratio,
                                                                                random_state=0)

            # Write the first header rwo to the result file
            metrics_path = "{}/{}.csv".format(self.model_metrics_path, data.opunit.name.lower())
            prediction_path = "{}/{}_prediction.csv".format(self.model_metrics_path, data.opunit.name.lower())
            result_writing_util.create_metrics_and_prediction_files(metrics_path, prediction_path)

            methods = self.ml_models
            # Only use linear regression for the arithmetic operating units
            if data.opunit in data_info.ARITHMETIC_OPUNITS:
                methods = ["lr"]

            # Also test the prediction with the target transformer (if specified for the operating unit)
            transformers = [None]
            modeling_transformer = data_transforming_util.OPUNIT_MODELING_TRANSFORMER_MAP[data.opunit]
            if modeling_transformer is not None:
                transformers.append(modeling_transformer)

            min_percentage_error = 1
            pred_results = None
            elapsed_us_index = data_info.TARGET_CSV_INDEX[Target.ELAPSED_US]

            for transformer in transformers:
                for method in methods:
                    # Train the model
                    logging.info("{} {}".format(data.opunit.name, method))
                    regressor = model.Model(method, modeling_transformer=transformer)
                    regressor.train(x_train, y_train)

                    # Evaluate on both the training and test set
                    results = []
                    evaluate_data = [(x_train, y_train), (x_test, y_test)]
                    train_test_label = ["Train", "Test"]
                    for i, d in enumerate(evaluate_data):
                        evaluate_x = d[0]
                        evaluate_y = d[1]

                        y_pred = regressor.predict(evaluate_x)
                        logging.debug("x shape: {}".format(evaluate_x.shape))
                        logging.debug("y shape: {}".format(y_pred.shape))
                        percentage_error = np.average(np.abs(evaluate_y - y_pred) / (evaluate_y + 1), axis=0)
                        results += list(percentage_error) + [""]

                        logging.info('{} Percentage Error: {}'.format(train_test_label[i], percentage_error))

                        # Record the model with the lowest elapsed time prediction (since that might be the most
                        # important prediction)
                        if (i == 1 and percentage_error[elapsed_us_index] < min_percentage_error and transformer ==
                                transformers[-1]):
                            min_percentage_error = percentage_error[elapsed_us_index]
                            model_map[data.opunit] = regressor
                            pred_results = (evaluate_x, y_pred, evaluate_y)

                    # Dump the prediction results
                    transform = " "
                    if transformer is not None:
                        transform = " transform"
                    io_util.write_csv_result(metrics_path, method + transform, results)

                    logging.info("")

                io_util.write_csv_result(metrics_path, "", [])

            # Record the best prediction results on the test data
            result_writing_util.record_predictions(pred_results, prediction_path)

        return model_map

# ==============================================
# main
# ==============================================
if __name__ == '__main__':
    aparser = argparse.ArgumentParser(description='Mini Trainer')
    aparser.add_argument('--input_path', default='mini_runner_input', help='Input file path for the mini runners')
    aparser.add_argument('--model_results_path', default='mini_runner_model_results',
                         help='Prediction results of the mini models')
    aparser.add_argument('--save_path', default='trained_model', help='Path to save the mini models')
    aparser.add_argument('--ml_models', nargs='*', type=str, default=["lr", "rf", "nn"],
                         help='ML models for the mini trainer to evaluate')
    aparser.add_argument('--test_ratio', type=float, default=0.2, help='Test data split ratio')
    aparser.add_argument('--log', default='info', help='The logging level')
    args = aparser.parse_args()

    logging_util.init_logging(args.log)

    trainer = MiniTrainer(args.input_path, args.model_results_path, args.ml_models, args.test_ratio)
    trained_model_map = trainer.train()
    with open(args.save_path + '/mini_model_map.pickle', 'wb') as file:
        pickle.dump(trained_model_map, file)


text:
from datetime import datetime

import numpy
import pytest

import wryte
from wryte import Wryte

class TestPerf(object):
    def _test_simple_message(self):
        w = Wryte(color=False, simple=True)

        timing = []

        for _ in range(5):
            now = datetime.now()

            for _ in range(10):
                w.info('My Message')

            timing.append((datetime.now() - now).total_seconds() * 1000.0)

        # This is just a benchmark. This should NEVER take this long.
        assert numpy.average(timing[1:]) < 10

    def test_simple_context(self):
        w = Wryte(color=False)

        timing = []

        for _ in range(5):
            now = datetime.now()

            for _ in range(10):
                w.info('My Message', {'key': 'value'})

            timing.append((datetime.now() - now).total_seconds() * 1000.0)

        # This is just a benchmark. This should NEVER take this long.
        assert numpy.average(timing[1:]) < 10


text:
# coding: utf-8

import numpy as np

sample_sizes = [3, 5, 10, 50]
n_samples = 5000000

bins = {}
bin_edges = np.arange(-2.5, 2.7, 0.005)

for size in sample_sizes:

    rands = np.random.uniform(low=-1.0, high=1.0, size=(n_samples, size))
    avgs = np.mean(rands, axis=1)
    bins[size], edges = np.histogram(avgs, bins=bin_edges, density=True)

with open("uniform.out", "w") as f:
    f.write("# Bin, counts (M=3,5,10,50)\n")
    for i in range(len(bin_edges) - 1):
        bin_center = (bin_edges[i] + bin_edges[i + 1]) / 2.0
        f.write("{:10.4f}".format(bin_center))
        for size in sample_sizes:
            f.write("{:10.5f}".format(bins[size][i]))
        f.write("\n")

######################################################################
######################################################################

# sample_sizes = [3,5,10,50]
# n_samples = 500000
#
# bins = {}
# bin_edges = np.arange(-2.5,2.7,0.1)
#
# for size in sample_sizes:
#
#    rands = np.random.uniform(size=(n_samples,size))
#    avgs = []
#    for j in range(rands.shape[0]):
#        isum=0
#        for i in range(rands.shape[1]):
#            if rands[j][i] < (1./2.):
#                val = rands[j][i]/2. - 2.
#            elif rands[j][i] < (3./4.):
#                val = rands[j][i]
#            else:
#                val = rands[j][i]/2. + 1.
#            isum += val
#        avgs.append(isum/size)
#
#    bins[size], edges = np.histogram(avgs,bins=bin_edges,density=True)
#
# with open('non-uniform.out','w') as f:
#    f.write('# Bin, counts (M=3,5,10,50)\n')
#    for i in range(len(bin_edges)-1):
#        bin_center = (bin_edges[i] + bin_edges[i+1])/2.
#        f.write('{:10.4f}'.format(bin_center))
#        for size in sample_sizes:
#            f.write('{:10.5f}'.format(bins[size][i]))
#        f.write('\n')
#
#


text:
from sympy.parsing.sympy_parser import parse_expr
from math import *
from sympy import *
import numpy as np

# Left endpoints integral approximation
def left(expr, left_bound, right_bound, delta_x):
    x = symbols('x')
    expression = parse_expr(expr)
    sum = 0
    for i in np.arange(left_bound, right_bound, delta_x):
        sum += expression.subs(x, i)
    return delta_x * sum

# RIght endpoints integral approximation
def right(expr, left_bound, right_bound, delta_x):
    x = symbols('x')
    expression = parse_expr(expr)
    sum = 0
    for i in np.arange(left_bound + delta_x, right_bound + delta_x, delta_x):
        sum += expression.subs(x, i)
    return delta_x * sum

# Midpoints integral approximation
def mid(expr, left_bound, right_bound, delta_x):
    x = symbols('x')
    expression = parse_expr(expr)
    sum = 0
    for i in np.arange(left_bound, right_bound, delta_x):
        sum += expression.subs(x, i + delta_x / 2)
    return delta_x * sum

# Trapezoidal Rule for integral approximation
def trapezoidal(expr, left_bound, right_bound, delta_x):
    return (left(expr, left_bound, right_bound, delta_x) \
            + right(expr, left_bound, right_bound, delta_x)) / 2

# Simpson's Rule for integral approximation
def simpson(expr, left_bound, right_bound, delta_x):
    x = symbols('x')
    expression = parse_expr(expr)
    sum = 0
    numbers = list(np.arange(left_bound, right_bound + delta_x, delta_x))
    for i in range(len(numbers)):
        if i == 0 or i == len(numbers) - 1:
            sum += expression.subs(x, numbers[i])
        elif i % 2 == 0:
            sum += 2 * expression.subs(x, numbers[i])
        else:
            sum += 4 * expression.subs(x, numbers[i])
    return delta_x * sum / 3

if __name__ == '__main__':
    # Read input
    expr = input('Expression: ')
    left_bound = float(input('Left Bound: '))
    right_bound = float(input('Right Bound: '))
    delta_x = float(input('Delta x: '))
    type = int(input('Select the type of approximation:\n \
    1. Left Endpoints\n \
    2. Right Endpoints\n \
    3. Midpoints\n \
    4. Trapezoidal Rule\n \
    5. Simpson\'s Rule\n'))
    # Determine which function to call
    if type == 1:
        print('Result:', left(expr, left_bound, right_bound, delta_x))
    elif type == 2:
        print('Result:', right(expr, left_bound, right_bound, delta_x))
    elif type == 3:
        print('Result:', mid(expr, left_bound, right_bound, delta_x))
    elif type == 4:
        print('Result:', trapezoidal(expr, left_bound, right_bound, delta_x))
    elif type == 5:
        print('Result:', simpson(expr, left_bound, right_bound, delta_x))

reason: import numpy,from sympy


text:
import os
import sys
import utils
import glob
import numpy as np
import cv2 as cv
from PIL import Image
import copy
import random
from scipy.sparse import coo_matrix
from scipy.sparse.csgraph import minimum_spanning_tree, connected_components, breadth_first_order
from pathlib import Path
import torch
from options.test_options import TestOptions
from models import create_model
from data.base_dataset import BaseDataset, get_transform

SAVE_DIR = sys.argv[1]
SKETCH_DIR = sys.argv[2]
OUT_DIR = sys.argv[3]
ERD_DIL = int(sys.argv[4])

sys.argv = [sys.argv[0]]

svp = Path(SAVE_DIR)
CHECKPOINTS_DIR = svp.parent

if not os.path.exists(OUT_DIR):
	os.makedirs(OUT_DIR)

def  get_corner_close_to_neigh(cx, cy, dx, dy, offset):

	if cx < dx:
		corner = [dx-offset, cy]

	if cx > dx:
		corner = [dx+offset, cy]

	if cy < dy:
		corner = [cx, dy-offset]

	if cy > dy:
		corner = [cx, dy+offset]

	return corner

def translate_patch(hybrid, cmask, patch_id, loc, model):

	transform = get_transform(opt, grayscale=True)

	hybrid_img = utils.ndarrayToPilImage(hybrid)
	hybrid_img = transform(hybrid_img)
	model_input = {'unstyled': hybrid_img, 'hybrid': hybrid_img, 'mask': cmask}
	model.set_input(model_input)
	model.test()

	result = model.fake
	im = utils.tensor2im(result)
	cres = np.asarray(im)

	return cres

def run_one_order(padded_input, order, adj, loc, all_patches, patch_size, model, context, final_outdir, style_name, disp_name, binary_thresh):

	styledResults = {}

	full_destyled = (patch_size - context) * (patch_size - context)

	nump = len(all_patches)
	rf_score = np.zeros((nump,1))

	h,w = padded_input.shape
	res = copy.deepcopy(padded_input)
	mask = np.ones([h, w])
	transform = get_transform(opt, grayscale=True)
	all_can = []
	all_can.append(copy.deepcopy(255-res))

	adj += adj.transpose()
	visited = np.zeros((adj.shape[0]), dtype=int)
	margin = []

	for idx,o in enumerate(order):

		if idx == 0:
			first = o

		visited[o] = 1
		tmp = adj[o,:]
		nidx = [x for x in range(len(tmp)) if tmp[x] > 0]
		noverlap = tmp[nidx]
		vnidx = visited[nidx]
		vnidx2 = [ix for ix,x in enumerate(vnidx) if x == 1]
		o_dep = np.asarray([nidx[index] for index in vnidx2])
		o_dep_overlap = np.asarray([noverlap[index] for index in vnidx2])

		curPatch = all_patches[o]
		cloc = loc[o]
		c = cloc

		hybrid = res[c[0]:c[0]+patch_size, c[1]:c[1]+patch_size]
		cmask = mask[c[0]:c[0]+patch_size, c[1]:c[1]+patch_size]

		unstyled = hybrid[cmask == 1]
		nz = np.count_nonzero(unstyled)

		if nz == 0:
			margin = margin + [o]
			mask[c[0]:c[0]+patch_size, c[1]:c[1]+patch_size] = -1
			continue

		if nz / full_destyled > 0.95:

			if len(o_dep) > 0:

				od = o_dep_overlap.argmax()
				od = o_dep[od]
				hybrid2 = copy.deepcopy(hybrid)
				c2 = copy.deepcopy(c)

				if od == o-1:
					hybrid2[:,patch_size-context:patch_size] = 0
					c2[1] = c[1] + context
				if od == o+1:
					hybrid2[:,0:context] = 0
					c2[1] = c[1] - context
				if od < o-1:
					hybrid2[patch_size-context:patch_size,:] = 0
					c2[0] = c[0] + context
				if od > o+1:
					hybrid2[0:context,:] = 0
					c2[0] = c[0] - context

				cres = translate_patch(hybrid2, cmask, o, c, model)

				x1,x2,y1,y2 = 0,0,0,0
				buf = round(context/2)
				buf2 = 6

				for dp in o_dep:
					if dp == o-1:
						y1 = buf
					if dp == o+1:
						y2 = -buf
					if dp < o-1:
						x1 = buf
					if dp > o+1:
						x2 = -buf

				if od == o-1:
					y2 = min(y2, -(context+buf2))
				if od == o+1:
					y1 = max(y1, context+buf2)
				if od < o-1:
					x2 = min(x2, -(context+buf2))
				if od > o+1:
					x1 = max(x1, context+buf2)

				res[c[0]+x1:c[0]+patch_size+x2, c[1]+y1:c[1]+patch_size+y2] = cres[x1:patch_size+x2, y1:patch_size+y2]

				htemp = res[c2[0]:c2[0]+patch_size, c2[1]:c2[1]+patch_size]
				hybrid3 = copy.deepcopy(htemp)

				if od == o-1:
					hybrid3[:,patch_size-context+buf2:patch_size] = 0
				if od == o+1:
					hybrid3[:,0:context-buf2] = 0
				if od < o-1:
					hybrid3[patch_size-context+buf2:patch_size,:] = 0
				if od > o+1:
					hybrid3[0:context-buf2,:] = 0

				cres = translate_patch(hybrid3, cmask, o, c, model)

				x1,x2,y1,y2 = 0,0,0,0

				for dp in o_dep:
					if dp == o-1:
						y1 = buf
					if dp == o+1:
						y2 = -buf
					if dp < o-1:
						x1 = buf
					if dp > o+1:
						x2 = -buf

				if od == o-1:
					y2 = min(y2, -(context+buf2))
				if od == o+1:
					y1 = max(y1, context+buf2)
				if od < o-1:
					x2 = min(x2, -(context+buf2))
				if od > o+1:
					x1 = max(x1, context+buf2)

				res[c2[0]+x1:c2[0]+patch_size+x2, c2[1]+y1:c2[1]+patch_size+y2] = cres[x1:patch_size+x2, y1:patch_size+y2]

				all_can.append(copy.deepcopy(255-res))

		cres = translate_patch(hybrid, cmask, o, c, model)

		x1,x2,y1,y2 = 0,0,0,0
		buf = round(context/2)

		for dp in o_dep:
			if dp == o-1:
				y1 = buf
			if dp == o+1:
				y2 = -buf
			if dp < o-1:
				x1 = buf
			if dp > o+1:
				x2 = -buf

		res[c[0]+x1:c[0]+patch_size+x2, c[1]+y1:c[1]+patch_size+y2] = cres[x1:patch_size+x2, y1:patch_size+y2]
		mask[c[0]:c[0]+patch_size, c[1]:c[1]+patch_size] = -1

		all_can.append(copy.deepcopy(255-res))

	for m in margin:

		tmp = adj[m,:]

		nidx = [x for x in range(len(tmp)) if tmp[x] > 0]
		noverlap = tmp[nidx]
		vnidx = visited[nidx]
		vnidx2 = [ix for ix,x in enumerate(vnidx) if x == 1]
		m_dep = np.asarray([nidx[index] for index in vnidx2])
		m_dep_overlap = np.asarray([noverlap[index] for index in vnidx2])

		for dix,dp in enumerate(m_dep):

			olap = m_dep_overlap[dix]
			if olap < 100:
				continue
			cloc = loc[m]
			dloc = loc[dp]
			cx = cloc[0]
			cy = cloc[1]
			dx = dloc[0]
			dy = dloc[1]

			corner = get_corner_close_to_neigh(cx, cy, dx, dy, context)
			c = corner
			hybrid = res[c[0]:c[0]+patch_size, c[1]:c[1]+patch_size]
			cmask = mask[c[0]:c[0]+patch_size, c[1]:c[1]+patch_size]
			unstyled = hybrid[cmask == 1]

			cres = translate_patch(hybrid, cmask, m, c, model)

			x1,x2,y1,y2 = 0,0,0,0
			buf = round(context/2)
			buf2 = round((patch_size - context) / 2)

			for dp2 in m_dep:
				cbuf = buf
				if dp2 == dp:
					cbuf = buf2

				if dp2 == o-1:
					y1 = cbuf
				if dp2 == o+1:
					y2 = -cbuf
				if dp2 < o-1:
					x1 = cbuf
				if dp2 > o+1:
					x2 = -cbuf

			res[c[0]+x1:c[0]+patch_size+x2, c[1]+y1:c[1]+patch_size+y2] = cres[x1:patch_size+x2, y1:patch_size+y2]
			mask[c[0]:c[0]+patch_size, c[1]:c[1]+patch_size] = -1

			all_can.append(copy.deepcopy(255-res))

	name_str = '%s_%s_%d' % (style_name, disp_name, first)
	file_name = os.path.join(final_outdir, name_str+'.png')
	samename = glob.glob(file_name)
	currnum = str(len(samename)+1)
	sname = os.path.join(final_outdir, name_str+'_'+currnum+'.png')
	print(sname)

	h,w = res.shape
	top, bot, left, right = utils.crop_image_by_bb(res, binary_thresh)
	pad = 4
	top = top - pad
	bot = bot + pad
	left = left - pad
	right = right + pad
	top = max(0, top)
	bot = min(h, bot)
	left = max(0, left)
	right = min(w, right)
	res = res[top:bot, left:right]

	utils.save_image(255-res, sname)
	utils.gen_gif(all_can, sname.replace('.png', '.gif'), 20, 0)

def run_one_sketch(sketch, model, style_name, mode='', num_iter=3):

	im_name = os.path.join(SKETCH_DIR, sketch)
	p = sketch.split('.')
	disp_name = '.'.join(p[:-1])

	final_outdir = OUT_DIR
	final_outdir = os.path.join(final_outdir, style_name)
	if not os.path.exists(final_outdir):
		os.makedirs(final_outdir)

	patch_size = 64
	context = int(patch_size/4)
	binary_thresh = 175

	im = utils.load_image(im_name)
	if mode == 'erode':
		im = utils.adjust_thickness(im, 1, num_iter)
		disp_name = disp_name + '_erd_%d' % num_iter
	if mode == 'dilate':
		im = utils.adjust_thickness(im, 0, num_iter)
		disp_name = disp_name + '_dil_%d' % num_iter
	all_patches, patch_nz, loc, num_patch_rows, num_patch_cols, padded_input = utils.image_to_patches(im, context, patch_size, binary_thresh)
	adj, non_empty = utils.calc_adj(all_patches, context, num_patch_rows, num_patch_cols)
	adjt = copy.deepcopy(adj)
	full_overlap = context * patch_size

	adjt = np.asarray(adjt)
	adjt[:,2] = full_overlap - adjt[:,2]
	numpa = len(all_patches)
	adj2 = coo_matrix((adjt[:,2], (adjt[:,0], adjt[:,1])), shape=(numpa, numpa))
	mst = minimum_spanning_tree(adj2)
	ncomp, lbls = connected_components(mst)
	adj = np.asarray(adj)
	adj = coo_matrix((adj[:,2], (adj[:,0], adj[:,1])), shape=(numpa, numpa))
	adj_mat = adj.toarray()
	lbls2 = lbls[non_empty]
	comps, counts = np.unique(lbls2, return_counts=True)

	fullpatch = patch_size ** 2

	num_to_gen = 1

	for i in range(num_to_gen):
		order = []
		dep = {}

		for idx,c in enumerate(comps):

			comp = c
			members = [x for x,y in enumerate(lbls) if y == comp]
			valid_members = [m for m in members if patch_nz[m] < 0.67 * fullpatch and patch_nz[m] > 0.1 * fullpatch]
			if len(valid_members) < 1:
				continue
			start = random.randrange(0,len(valid_members))
			start = valid_members[start]

			corder, pred = breadth_first_order(mst, start, directed=False)#62)
			order.extend(corder)

		run_one_order(padded_input, order, adj_mat, loc, all_patches, patch_size, model, context, final_outdir, style_name, disp_name, binary_thresh)

def run_one_model(topt, save_dir, erd_dil):

	style_name = os.path.basename(save_dir)
	topt.opt.save_dir = style_name
	opt = topt.post_parse()

	opt.nThreads = 1
	opt.batchSize = 1
	opt.no_flip = True
	opt.display_id = -1
	opt.model =  'sketchpatch'
	opt.no_dropout = True
	opt.load_size = 64
	opt.crop_size = 64
	opt.input_nc = 1
	opt.output_nc = 1

	torch.manual_seed(0)

	model = create_model(opt)
	if opt.eval:
		model.eval()
	model.setup(opt)

	sketches = [f for f in os.listdir(SKETCH_DIR) if f.lower().endswith(('.png','.jpg','jpeg'))]

	for sk in sketches:

		if erd_dil == 0:
			run_one_sketch(sk, model, style_name, '')

		if erd_dil < 0:
			try:
				run_one_sketch(sk, model, style_name, mode='erode', num_iter=-1*erd_dil)
			except:
				print("failed on erosion with num iter %d" % (-1*erd_dil))

		if erd_dil > 0:
			try:
				run_one_sketch(sk, model, style_name, mode='dilate', num_iter=erd_dil)
			except:
				print("failed on dilation with num iter %d" % erd_dil)

if __name__ == '__main__':

	topt = TestOptions()
	opt = topt.parse()
	topt.opt.checkpoints_dir = CHECKPOINTS_DIR
	run_one_model(topt, SAVE_DIR, ERD_DIL)

max_stars_repo_name: noafish/SketchPatch

max_issues_repo_name: noafish/SketchPatch

max_forks_repo_name: noafish/SketchPatch

reason: import numpy,from scipy


text:
#!/usr/bin/env python

# standard lib
import warnings

# local
from ..utils import PyDyUserWarning
from .shapes import Shape
from .visualization_frame import VisualizationFrame

__all__ = ['PerspectiveCamera', 'OrthoGraphicCamera']

warnings.simplefilter('once', PyDyUserWarning)

class PerspectiveCamera(VisualizationFrame):
    """Creates a perspective camera for use in a scene. The camera is inherited
    from ``VisualizationFrame``, and thus behaves similarly. It can be attached
    to dynamics objects, hence we can get a moving camera. All the
    transformation matrix generation methods are applicable to a
    ``PerspectiveCamera``.

    """

    def __init__(self, *args, **kwargs):
        """Initialises a PerspectiveCamera object. To initialize a
        PerspectiveCamera, one needs to supply a name (optional), a reference
        frame, a point, field of view (fov) (optional), near plane distance
        (optional) and far plane distance (optional).

        Like ``VisualizationFrame``, it can also be initialized using one of
        these three argument sequences:

        Rigidbody
        ``PerspectiveCamera(rigid_body)``
        ReferenceFrame, Point
        ``PerspectiveCamera(ref_frame, point)``
        ReferenceFrame, Particle
        ``PerspectiveCamera(ref_frame, particle)``

        Note that you can also supply and optional name as the first positional
        argument, e.g.::

           ``PerspectiveCamera('camera_name', rigid_body)``

        Additional optional keyword arguments are below:

        Parameters
        ==========
        fov : float, default=45.0
            Field Of View, It determines the angle between the top and bottom
            of the viewable area (in degrees).
        near : float
            The distance of near plane of the PerspectiveCamera. All objects
            closer to this distance are not displayed.
        far : int or float
            The distance of far plane of the PerspectiveCamera. All objects
            farther than this distance are not displayed.

        Examples
        ========

        >>> from sympy import symbols
        >>> from sympy.physics.mechanics import (ReferenceFrame, Point,
        ...                                      RigidBody, Particle,
        ...                                      inertia)
        >>> from pydy.viz import PerspectiveCamera
        >>> I = ReferenceFrame('I')
        >>> O = Point('O')

        >>> # initializing with reference frame, point
        >>> camera1 = PerspectiveCamera('frame1', I, O)

        >>> # Initializing with a RigidBody
        >>> Ixx, Iyy, Izz, mass = symbols('Ixx Iyy Izz mass')
        >>> i = inertia(I, Ixx, Iyy, Izz)
        >>> rbody = RigidBody('rbody', O, I, mass, (inertia, O))
        >>> camera2 = PerspectiveCamera('frame2', rbody)

        >>> # initializing with Particle, reference_frame
        >>> Pa = Particle('Pa', O, mass)
        >>> camera3 = PerspectiveCamera('frame3', I, Pa)

        """

        # NOTE: This allows us to use inhertiance even though cameras don't
        # need a shape. In the future, this could be a camera shape that could
        # be made visible in the scene (only important for multiple cameras).
        args = list(args) + [Shape()]

        super(PerspectiveCamera, self).__init__(*args)

        self.fov = 45.0
        self.near = 1.0
        self.far = 1000.0

        for k, v in kwargs.items():
            setattr(self, k, v)

    def __str__(self):
        return 'PerspectiveCamera: ' + self.name

    def __repr__(self):
        return 'PerspectiveCamera'

    @property
    def fov(self):
        return self._fov

    @fov.setter
    def fov(self, new_fov):
        self._fov = float(new_fov)

    @property
    def near(self):
        return self._near

    @near.setter
    def near(self, new_near):
        self._near = float(new_near)

    @property
    def far(self):
        return self._far

    @far.setter
    def far(self, new_far):
        self._far = float(new_far)

    def generate_scene_dict(self, **kwargs):
        """This method generates information for a static visualization in the
        initial conditions, in the form of dictionary. This contains camera
        parameters followed by an init_orientation key.

        Before calling this method, all the transformation matrix generation
        methods should be called, or it will give an error.

        Returns
        =======
        A dict with following Keys:

        1. name: name for the camera
        2. fov: Field of View value of the camera
        3. near: near value of the camera
        4. far: far value of the camera
        5. init_orientation: Initial orientation of the camera

        """
        scene_dict = super(PerspectiveCamera, self).generate_scene_dict(**kwargs)
        scene_dict[id(self)]['type'] = self.__class__.__name__
        scene_dict[id(self)]['fov'] = self.fov
        scene_dict[id(self)]['near'] = self.near
        scene_dict[id(self)]['far'] = self.far

        return scene_dict

class OrthoGraphicCamera(VisualizationFrame):
    """Creates a orthographic camera for use in a scene. The camera is
    inherited from ``VisualizationFrame``, and thus behaves similarly. It can
    be attached to dynamics objects, hence we can get a moving camera. All the
    transformation matrix generation methods are applicable to a
    ``OrthoGraphicCameraCamera``.

    """

    def __init__(self, *args, **kwargs):
        """Initialises a OrthoGraphicCameraCamera object. To initialize a
        OrthoGraphicCameraCamera, one needs to supply a name (optional), a
        reference frame, a point, field of view (fov) (optional), near plane
        distance (optional) and far plane distance (optional).

        Like ``VisualizationFrame``, it can also be initialized using one of
        these three argument sequences:

        Rigidbody
           ``OrthoGraphicCameraCamera(rigid_body)``
        ReferenceFrame, Point
           ``OrthoGraphicCameraCamera(ref_frame, point)``
        ReferenceFrame, Particle
           ``OrthoGraphicCameraCamera(ref_frame, particle)``

        Note that you can also supply and optional name as the first positional
        argument, e.g.::

            OrthoGraphicCameraCamera('camera_name', rigid_body)

        Additional optional keyword arguments are below:

        Parameters
        ==========
        near : float
            The distance of near plane of the OrthoGraphicCameraCamera. All
            objects closer to this distance are not displayed.
        far : int or float
            The distance of far plane of the OrthoGraphicCameraCamera. All
            objects farther than this distance are not displayed.

        Examples
        ========

        >>> from sympy import symbols
        >>> from sympy.physics.mechanics import (ReferenceFrame, Point,
        ...                                      RigidBody, Particle,
        ...                                      inertia)
        >>> from pydy.viz import OrthoGraphicCameraCamera
        >>> I = ReferenceFrame('I')
        >>> O = Point('O')

        >>> # initializing with reference frame, point
        >>> camera1 = OrthoGraphicCameraCamera('frame1', I, O)

        >>> # Initializing with a RigidBody
        >>> Ixx, Iyy, Izz, mass = symbols('Ixx Iyy Izz mass')
        >>> i = inertia(I, Ixx, Iyy, Izz)
        >>> rbody = RigidBody('rbody', O, I, mass, (inertia, O))
        >>> camera2 = OrthoGraphicCameraCamera('frame2', rbody)

        >>> # initializing with Particle, reference_frame
        >>> Pa = Particle('Pa', O, mass)
        >>> camera3 = OrthoGraphicCameraCamera('frame3', I, Pa)

        """
        # NOTE: This allows us to use inhertiance even though cameras don't
        # need a shape. In the future, this could be a camera shape that could
        # be made visible in the scene (only important for multiple cameras).
        args = list(args) + [Shape()]

        super(OrthoGraphicCamera, self).__init__(*args)

        self.near = 1.0
        self.far = 1000.0

        for k, v in kwargs.items():
            setattr(self, k, v)

    def __str__(self):
        return self.__class__.__name__ + ': ' + self.name

    def __repr__(self):
        return self.__class__.__name__

    @property
    def near(self):
        return self._near

    @near.setter
    def near(self, new_near):
        self._near = float(new_near)

    @property
    def far(self):
        return self._far

    @far.setter
    def far(self, new_far):
        self._far = float(new_far)

    def generate_scene_dict(self, **kwargs):
        """
        This method generates information for a static visualization in the
        initial conditions, in the form of dictionary. This contains camera
        parameters followed by an init_orientation Key.

        Returns
        =======
        scene_dict : dictionary
           A dict with following Keys:

              1. name: name for the camera
              2. near: near value of the camera
              3. far: far value of the camera
              4. init_orientation: Initial orientation of the camera

        """
        scene_dict = super(OrthoGraphicCamera, self).generate_scene_dict(**kwargs)
        scene_dict[id(self)]['type'] = self.__class__.__name__
        scene_dict[id(self)]['near'] = self.near
        scene_dict[id(self)]['far'] = self.far

        return scene_dict

max_stars_repo_name: JanMV/pydy

max_issues_repo_name: JanMV/pydy

max_forks_repo_name: JanMV/pydy

reason: from sympy


text:
"""
Tests for bias solving routines
"""
from dataclasses import dataclass
import random
from tid import bias_solve
from typing import cast, Any, Dict, List

import numpy

from laika.lib import coordinates

@dataclass
class ConnTickMap:
    connections: List

@dataclass
class FakeConnection:
    station: str
    prn: str
    ticks: numpy.ndarray
    ipps: numpy.ndarray
    vtecs: numpy.ndarray
    is_glonass: bool
    glonass_chan: int

@dataclass
class FakeScenario:
    conn_map: Dict

def generate_data(station_count=4, sat_count=4, duration=240):
    """
    Args:
        station_count: number of fake stations
        sat_count: number of fake satellites
        duration: total tick number

    Returns:
        FakseScenario
    """
    stations = [f"station_{i}" for i in range(station_count)]
    sats = [f"sat_{i}" for i in range(sat_count)]
    sat_glonass = {sat: random.choice([None, -3, 2, 6]) for sat in sats}

    station_biases = dict(
        zip(stations, (numpy.random.rand(station_count, 3) - 0.5) * 20)
    )
    sat_biases = dict(zip(sats, (numpy.random.rand(sat_count) - 0.5) * 20))

    # populate 16 places where ipps can occur, measurements will randomly
    # be spread out among them
    lats, lons = numpy.meshgrid(numpy.arange(20, 40, 5), numpy.arange(20, 40, 5))
    coords = numpy.stack(
        (numpy.reshape(lats, 16), numpy.reshape(lons, 16), numpy.zeros(16)), axis=1
    )

    # ipps from connections are ECEF XYZ
    ipps_allowed = coordinates.geodetic2ecef(coords)

    # pick TECu values for each IPP and each tick
    TEC_truths = numpy.random.rand(duration, 16) * 50

    connection_data = cast(Dict[str, Dict[str, Any]], {})
    for station in stations:
        connection_data[station] = dict()
        for sat in sats:
            ipps = numpy.zeros((duration, 3))
            vtecs = numpy.zeros((duration, 2))
            ticks = numpy.arange(duration)

            is_glonass = sat_glonass[sat] is not None
            glonass_chan = sat_glonass[sat] or 0

            for tick in ticks:
                # pick a location
                loc = random.randint(0, 15)
                # pick a slant
                slant = numpy.random.rand() * 0.5 + 0.25

                TEC = TEC_truths[tick][loc]
                ipps[tick] = ipps_allowed[loc]

                if not is_glonass:
                    station_bias = station_biases[station][0]
                else:
                    station_bias = (
                        station_biases[station][1]
                        + station_biases[station][2] * glonass_chan
                    )
                measurement = TEC + (station_bias - sat_biases[sat]) * slant
                vtecs[tick] = (measurement, slant)

            connection_data[station][sat] = ConnTickMap(
                [
                    FakeConnection(
                        station, sat, ticks, ipps, vtecs.T, is_glonass, glonass_chan
                    )
                ]
            )

    return FakeScenario(connection_data), sat_biases, station_biases

def true_error(
    true_sat_biases, true_station_biases, calcd_sat_biases, calcd_station_biases
):
    """
    How far off were the biases from the true biases?
    """

    sum_squared_error = 0.0
    for station in true_station_biases:
        sum_squared_error += cast(
            float,
            numpy.linalg.norm(
                true_station_biases[station] - calcd_station_biases[station]
            ),
        )

    for sat in true_sat_biases:
        sum_squared_error += (true_sat_biases[sat] - calcd_sat_biases[sat]) ** 2

    return sum_squared_error

def bias_solve_error(station_count=10, sat_count=10, duration=240):
    """
    Args:
        station_count: number of fake stations
        sat_count: number of fake satellites
        duration: total tick number

    Returns:
        mean squared error in all bias calculations
    """
    fake_sc, sat_biases, station_biases = generate_data(
        station_count, sat_count, duration
    )
    bsolver = bias_solve.SimpleBiasSolver(fake_sc)
    calcd_sat_biases, calcd_station_biases = bsolver.solve_biases()

    return true_error(
        sat_biases, station_biases, calcd_sat_biases, calcd_station_biases
    ) / (len(sat_biases) + len(station_biases) * 3)

def test_bias_solver():
    """
    Run bias solver 200 times with simple simulations. Verify that the mean
    squared error for parameters is <3.0 50% of the time, and <6.0 90% of the time
    """
    errors = [bias_solve_error(duration=1) for _ in range(200)]
    assert numpy.quantile(errors, 0.5) < 3.0
    assert numpy.quantile(errors, 0.9) < 6.0

max_stars_repo_name: BenMueller/missile-tid

max_issues_repo_name: BenMueller/missile-tid

max_forks_repo_name: BenMueller/missile-tid


text:
# Copyright 2020 Jan Jitse Venselaar.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import torch
from time import time
import torch.nn as nn
import torch.optim as optim
from tqdm import tqdm
import numpy as np
from tensorboardX import SummaryWriter
import torchvision.utils as vutils
import logging
import argparse
from pathlib import Path
from typing import Tuple, List

from models import BasicNet, DeeperNet
from loaders import loader_mnist, loader_fmnist
from samplebuffer import SampleBuffer
from evaluate import evaluate_acc, generate_unconditional

RANDOM_SEED = 42

torch.manual_seed(RANDOM_SEED)

def sgld(
    model: nn.Module,
    buffer: SampleBuffer,
    nr_samples: int,
    pcd_length: int,
    alpha: float,
    sigma: float,
    random_reinit_freq: float,
) -> Tuple[torch.Tensor, List]:
    samples, classes = buffer.sample(nr_samples, random_reinit_freq)
    samples = samples[torch.randperm(nr_samples)]
    model.eval()
    samples = samples.to(device)
    energies = []
    x_t = samples.detach().requires_grad_(True)
    for t in range(pcd_length):
        energy = -model(x_t).logsumexp(dim=1).sum()
        x_t.data += -alpha * torch.autograd.grad(energy, x_t)[
            0
        ] + sigma * torch.randn_like(x_t)
        # TODO: somehow normalize based on this data?
        energies.append(energy.detach().cpu().numpy())
    samples = x_t.detach()
    model.train()
    buffer.push(samples, classes)
    return samples, energies

def save_checkpoint(model: nn.Module, buffer: SampleBuffer, save_dir: Path, tag):
    checkpoint_dict = {"model_state": model.state_dict(), "sample_buffer": buffer}
    torch.save(checkpoint_dict, save_dir / tag)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--cuda", default=True, type=bool)
    parser.add_argument("--dataset", default="MNIST", choices=["MNIST", "FMNIST"])
    parser.add_argument("--lr", default=1e-4, type=float)
    parser.add_argument("--n_epochs", default=20, type=int)
    parser.add_argument("--chain_length", default=20, type=int)
    parser.add_argument("--reinit_rate", default=0.05, type=float)
    parser.add_argument("--alpha", default=1.0, type=float)
    parser.add_argument("--sigma", default=0.01, type=float)
    parser.add_argument("--batch_size", default=64, type=int)
    parser.add_argument("--buffer_size", default=1e4, type=int)
    parser.add_argument("--image_noise", default=0.01, type=float)
    parser.add_argument("--save_dir", default=".", type=str)

    args = parser.parse_args()

    REINIT_FREQ_START = 0.05
    REINIT_DECAY_RATE = 1.0
    REINIT_MIN = 0.05

    if args.dataset == "MNIST":
        train_loader = loader_mnist(args.batch_size, image_noise=args.image_noise)
        test_loader = loader_mnist(args.batch_size, train=False)
    elif args.dataset == "FMNIST":
        train_loader = loader_fmnist(args.batch_size, image_noise=args.image_noise)
        test_loader = loader_fmnist(args.batch_size, train=False)
    reinit_freq = args.reinit_rate

    if args.cuda:
        device = torch.device("cuda")
        torch.backends.cudnn.enabled = True
    else:
        device = torch.device("cpu")

    image_shape = iter(train_loader).next()[0].shape[1:]

    network = BasicNet(input_shape=image_shape)
    network.to(device)

    optimizer = optim.SGD(network.parameters(), lr=args.lr)
    samplebuffer = SampleBuffer(max_samples=args.buffer_size)
    objective_clf = nn.CrossEntropyLoss()
    writer = SummaryWriter()

    for e in range(args.n_epochs):
        time0 = time()
        reinit_freq = max(REINIT_MIN, reinit_freq * REINIT_DECAY_RATE)
        losses_clf = []
        losses_gen = []
        energy_list = []
        for images, labels in tqdm(train_loader):
            images = images.to(device, non_blocking=True)
            labels = labels.to(device, non_blocking=True)

            samples, energies = sgld(
                network,
                samplebuffer,
                images.shape[0],
                args.chain_length,
                args.alpha,
                args.sigma,
                reinit_freq,
            )
            output = network(images)
            loss_clf = objective_clf(output, labels)
            loss_gen = -torch.mean(
                torch.logsumexp(output, dim=1)
                - torch.logsumexp(network(samples), dim=1)
            )
            loss_total = loss_clf + loss_gen
            optimizer.zero_grad()
            loss_total.backward()
            optimizer.step()
            losses_clf.append(loss_clf.detach().cpu().numpy())
            losses_gen.append(loss_gen.detach().cpu().numpy())
            energy_list.append(energies)
        else:
            acc = evaluate_acc(test_loader, network)
            logging.info(
                f"Epoch: {e}, classification accuracy: {acc}, time taken {time() - time0:.2f}, reinit freq: {reinit_freq}, generator loss: {np.mean(losses_gen):.2f}, classifier loss: {np.mean(losses_clf):.2f}, mean energy: {np.mean(energies):.2f}."
            )
            writer.add_scalar("accuracy", acc, e)
            writer.add_scalar("generator_loss", np.mean(losses_gen), e)
            writer.add_scalar("classifier_loss", np.mean(losses_clf), e)
            writer.add_scalar("energy", np.mean(energies), e)
            examples, _ = samplebuffer.sample(5, reinit_freq=0)
            writer.add_image("generated", vutils.make_grid(examples.data), e)
            writer.add_image("real", vutils.make_grid(images.data[:5]), e)
            save_checkpoint(
                network, samplebuffer, Path(args.save_dir), "last_checkpoint.pt"
            )

max_stars_repo_name: janjitse/Energy-Based-Models

max_issues_repo_name: janjitse/Energy-Based-Models

max_forks_repo_name: janjitse/Energy-Based-Models


text:
import os.path
import torchvision.transforms as transforms
from data.base_dataset import BaseDataset, get_transform, get_random_trans
from data.image_folder import make_dataset
from PIL import Image
import PIL
import random
import pandas as pd
import numpy as np
import torch
import torchvision.transforms as transforms

class KeyDataset(BaseDataset):
    def initialize(self, opt):
        self.opt = opt
        self.root = opt.dataroot
        self.dir_P = os.path.join(opt.dataroot, opt.phase)  # person images
        self.dir_K = os.path.join(opt.dataroot, opt.phase + 'K')  # keypoints

        self.init_categories(opt.pairLst)
        self.transform = get_transform(opt)

    def init_categories(self, pairLst):
        pairs_file_train = pd.read_csv(pairLst)
        self.size = len(pairs_file_train)
        self.pairs = []
        print('Loading data pairs ...')
        for i in range(self.size):
            pair = [pairs_file_train.iloc[i]['from'], pairs_file_train.iloc[i]['to']]
            self.pairs.append(pair)

        print('Loading data pairs finished ...')

    def __getitem__(self, index):
        # self.opt.random = True
        if self.opt.phase == 'train' or self.opt.random:
            index = random.randint(0, self.size - 1)

        P1_name, P2_name = self.pairs[index]

        if self.opt.debug:
            # while 'Transpolis' not in P1_name and 'Laure' not in P1_name and 'Chama' not in P1_name:
            while 'Transpolis' in P1_name or 'Laure' in P1_name or 'Chama' in P1_name:
                if self.opt.phase == 'train' or self.opt.random:
                    index = random.randint(0, self.size - 1)

                P1_name, P2_name = self.pairs[index]

        P1_path = os.path.join(self.dir_P, P1_name)  # person 1
        BP1_path = os.path.join(self.dir_K, P1_name + '.npy')  # bone of person 1

        # person 2 and its bone
        P2_path = os.path.join(self.dir_P, P2_name)  # person 2
        BP2_path = os.path.join(self.dir_K, P2_name + '.npy')  # bone of person 2

        P1_img = Image.open(P1_path).convert('RGB')
        P2_img = Image.open(P2_path).convert('RGB')

        BP1_img = np.load(BP1_path)  # h, w, c
        BP2_img = np.load(BP2_path)

        # P1_img = Image.new('RGB', P1_img.size)
        # P2_img = Image.new('RGB', P2_img.size)
        # BP2_img = np.empty_like(BP2_img)
        # use flip
        if self.opt.phase == 'train':
            # print ('use_flip ...')
            flip_random = random.uniform(0, 1)
            rs_random = random.uniform(0, 1)
            if rs_random > 0.6 and not self.opt.no_rotate:
                rs_transform = get_random_trans(P1_img.size[:2])

                P1_img = rs_transform(P1_img)
                P2_img = rs_transform(P2_img)

                BP1_img = rs_transform(BP1_img)
                BP2_img = rs_transform(BP2_img)

            if flip_random > 0.5 and not self.opt.no_flip:
                # print('fliped ...')
                P1_img = P1_img.transpose(Image.FLIP_LEFT_RIGHT)
                P2_img = P2_img.transpose(Image.FLIP_LEFT_RIGHT)

                BP1_img = flip_keypoints(BP1_img)  # flip
                BP2_img = flip_keypoints(BP2_img)  # flip

        BP1 = torch.from_numpy(BP1_img).float()  # h, w, c
        BP1 = BP1.transpose(2, 0)  # c,w,h
        BP1 = BP1.transpose(2, 1)  # c,h,w

        BP2 = torch.from_numpy(BP2_img).float()
        BP2 = BP2.transpose(2, 0)  # c,w,h
        BP2 = BP2.transpose(2, 1)  # c,h,w

        P1 = self.transform(P1_img)
        P2 = self.transform(P2_img)

        return {'P1': P1, 'BP1': BP1, 'P2': P2, 'BP2': BP2,
                'P1_path': P1_name, 'P2_path': P2_name}

    def __len__(self):
        if self.opt.phase == 'train':
            return self.opt.epoch_size
        else:
            return self.size

    def name(self):
        return 'KeyDataset'

def flip_keypoints(bp):
    bp = np.array(bp[:, ::-1, :])
    idxs = [0, 1, 5, 6, 7, 2, 3, 4, 11, 12, 13, 8, 9, 10, 15, 14, 17, 16] + list(range(18, bp.shape[-1]))
    bp = bp[:, :, idxs]
    return bp

max_stars_repo_name: Archjbald/PoseStylizer

max_issues_repo_name: Archjbald/PoseStylizer

max_forks_repo_name: Archjbald/PoseStylizer


text:
import numpy as np

from src.emission import MLE_emission_parameters, new_MLE_emission_parameters_with_unknown, predict_y
from src.transition import MLE_transition_parameters
from src.viterbi import viterbi
from src.best_k_viterbi import get_stateset_and_wordset, convert_to_log_nonsparse_emission_dict, convert_to_log_nonsparse_tranmission_dict
from src.part_3 import part_3

folder_dir = "ES"
train_dir = f"data/{folder_dir}/train"
test_dir = f"data/{folder_dir}/dev.in"

if __name__ == "__main__":
    # Part 1

    # Find emission params
    count_y_dict, count_y_to_x_dict, emission_dict = MLE_emission_parameters(train_dir)
    emission_dict = new_MLE_emission_parameters_with_unknown(count_y_dict, count_y_to_x_dict, emission_dict, k=1)

    p1_output_dir = f"data/{folder_dir}/dev.p1.out"

    # Predict Sequence labels
    predict_y(emission_dict,test_dir,p1_output_dir) # Output in output_dir

    # Part 2

    # Find transition paramters
    count_y_dict, count_y_to_y_dict, transition_dict = MLE_transition_parameters(train_dir)

    p2_output_dir = f"data/{folder_dir}/dev.p2.out"

    # Perform Viterbi
    viterbi(emission_dict, transition_dict, test_dir, p2_output_dir)

    # Part 3
    stateset, wordset = get_stateset_and_wordset(emission_dict,transition_dict)
    nonsparse_emission_dict = convert_to_log_nonsparse_emission_dict(emission_dict,stateset,wordset)
    nonsparse_transmission_dict = convert_to_log_nonsparse_tranmission_dict(transition_dict,stateset)
    part_3(nonsparse_emission_dict,nonsparse_transmission_dict,stateset,test_dir)
    print(f"Done generating {folder_dir} outputs")

    # Part 4 exists in project_part_4.ipynb


text:
#!/usr/bin/env python

import click as ck
import numpy as np
import pandas as pd
from collections import Counter
from utils import Ontology, FUNC_DICT
import logging

logging.basicConfig(level=logging.INFO)

@ck.command()
@ck.option(
    '--go-file', '-gf', default='data/go.obo',
    help='Gene Ontology file in OBO Format')
@ck.option(
    '--hp-file', '-hf', default='data/hp.obo',
    help='Human Phenotype Ontology file in OBO Format')
@ck.option(
    '--hp-annots-file', '-haf', default='data/ALL_SOURCES_ALL_FREQUENCIES_genes_to_phenotype.txt',
    help='Human Phenotype Ontology annotations')
@ck.option(
    '--deepgo-annots-file', '-daf', default='data/human.res',
    help='Predicted go annotations with DeepGOPlus')
@ck.option(
    '--id-mapping-file', '-imf', default='data/gene2prot.tab',
    help='Uniprot KB, generated with uni2pandas.py')
@ck.option(
    '--data-file', '-df', default='data/swissprot.pkl',
    help='Uniprot KB, generated with uni2pandas.py')
@ck.option(
    '--string-mapping-file', '-smf', default='data/string2uni.tab',
    help='ELEmbeddings')
@ck.option(
    '--expressions-file', '-ef', default='data/E-MTAB-5214-query-results.tpms.tsv',
    help='ELEmbeddings')
@ck.option(
    '--out-terms-file', '-otf', default='data/terms.pkl',
    help='Terms for prediction')
@ck.option(
    '--out-data-file', '-odf', default='data/human.pkl',
    help='Data file')
@ck.option(
    '--min-count', '-mc', default=10,
    help='Min count of HP classes for prediction')
def main(go_file, hp_file, hp_annots_file, deepgo_annots_file, id_mapping_file,
         data_file, string_mapping_file, expressions_file,
         out_data_file, out_terms_file, min_count):
    go = Ontology(go_file, with_rels=True)
    print('GO loaded')
    hp = Ontology(hp_file, with_rels=True)
    print('HP loaded')
    print('Load Gene2prot mapping')
    df = pd.read_pickle(data_file)
    prot2gene = {}
    with open(id_mapping_file) as f:
        next(f)
        for line in f:
            it = line.strip().split('\t')
            prot2gene[it[2]] = it[0]
    # for row in df.itertuples():
    #     if row.genes != '':
    #         prot2gene[row.proteins] = row.genes

    print('Loading HP annotations')
    hp_annots = {}
    name2gene = {}
    with open(hp_annots_file) as f:
        next(f)
        for line in f:
            it = line.strip().split('\t')
            gene_id = it[0]
            gene_name = it[1].upper()
            name2gene[gene_name] = gene_id
            hp_id = it[3]
            if gene_id not in hp_annots:
                hp_annots[gene_id] = set()
            if hp.has_term(hp_id):
                hp_annots[gene_id] |= hp.get_anchestors(hp_id)
    total_annots = 0
    for g_id, annots in hp_annots.items():
        total_annots += len(annots)
    print('HP Annotations', len(hp_annots), total_annots, (total_annots / len(hp_annots)))
    dg_annots = {}
    gos = set()
    with open(deepgo_annots_file) as f:
        for line in f:
            it = line.strip().split('\t')
            if it[0] not in prot2gene:
                continue
            gene_id = prot2gene[it[0]]
            annots = dg_annots.get(gene_id, {})
            for item in it[1:]:
                go_id, score = item.split('|')
                score = float(score)
                annots[go_id] = max(score, annots.get(go_id, 0))
            dg_annots[gene_id] = annots
            gos |= set(annots.keys())
    print('DeepGO Annotations', len(dg_annots))
    deepgo_annots = {}
    for g_id, annots in dg_annots.items():
        deepgo_annots[g_id] = [go_id + '|' + str(score) for go_id, score in annots.items()]
    print('Number of GOs', len(gos))
    gos_df = pd.DataFrame({'gos': list(gos)})
    # gos_df.to_pickle('data/gos.pkl')

    go_annots = {}
    iea_annots = {}
    seqs = {}

    for i, row in df.iterrows():
        if row.proteins not in prot2gene:
            continue
        g_id = prot2gene[row.proteins]
        if g_id not in go_annots:
            go_annots[g_id] = set()
            iea_annots[g_id] = set()
        go_annots[g_id] |= set(row.exp_annotations)
        iea_annots[g_id] |= set(row.iea_annotations)
        seqs[g_id] = row.sequences

    print('GO Annotations', len(go_annots))
    logging.info('Processing annotations')

    cnt = Counter()
    annotations = list()
    for g_id, annots in hp_annots.items():
        for term in annots:
            cnt[term] += 1

    gene_exp = {}
    max_val = 0
    with open(expressions_file) as f:
        for line in f:
            if line.startswith('#') or line.startswith('Gene'):
                continue
            it = line.strip().split('\t')
            gene_name = it[1].upper()
            if gene_name in name2gene:
                exp = np.zeros((53,), dtype=np.float32)
                for i in range(len(it[2:])):
                    exp[i] = float(it[2 + i]) if it[2 + i] != '' else 0.0
                gene_exp[name2gene[gene_name]] = exp / np.max(exp)

    print('Expression values', len(gene_exp))

    deepgo_annotations = []
    go_annotations = []
    iea_annotations = []
    hpos = []
    genes = []
    sequences = []
    expressions = []
    mis_exp = 0
    for g_id, phenos in hp_annots.items():
        if g_id not in dg_annots:
            continue
        genes.append(g_id)
        hpos.append(phenos)
        go_annotations.append(go_annots[g_id])
        iea_annotations.append(iea_annots[g_id])
        deepgo_annotations.append(deepgo_annots[g_id])
        sequences.append(seqs[g_id])
        if g_id in gene_exp:
            expressions.append(gene_exp[g_id])
        else:
            expressions.append(np.zeros((53,), dtype=np.float32))
            mis_exp += 1
    print('Missing expressions', mis_exp)

    # for g_id, gos in dg_annots.items():
    #     genes.append(g_id)
    #     phenos = set()
    #     if g_id in hp_annots:
    #         phenos = hp_annots[g_id]
    #     hpos.append(phenos)
    #     go_annotations.append(go_annots[g_id])
    #     iea_annotations.append(iea_annots[g_id])
    #     deepgo_annotations.append(deepgo_annots[g_id])
    #     sequences.append(seqs[g_id])

    df = pd.DataFrame(
        {'genes': genes, 'hp_annotations': hpos,
         'go_annotations': go_annotations, 'iea_annotations': iea_annotations,
         'deepgo_annotations': deepgo_annotations,
         'sequences': sequences, 'expressions': expressions})
    df.to_pickle(out_data_file)
    print(f'Number of proteins {len(df)}')
    print(df)
    # Filter terms with annotations more than min_count
    terms_set = set()
    all_terms = []
    for key, val in cnt.items():
        if key == 'HP:0000001':
            continue
        all_terms.append(key)
        if val >= min_count:
            terms_set.add(key)
    terms = []
    labels = []
    for t_id in hp.get_ordered_terms():
        if t_id in terms_set:
            terms.append(t_id)
            labels.append(hp.get_term(t_id)['name'])

    logging.info(f'Number of terms {len(terms)}')
    # Save the list of terms
    # df = pd.DataFrame({'terms': terms, 'labels': labels})
    # df.to_pickle(out_terms_file)
    # df = pd.DataFrame({'terms': all_terms})
    # df.to_pickle('data/all_terms.pkl')

if __name__ == '__main__':
    main()

max_stars_repo_name: bio-ontology-research-group/deeppheno

max_issues_repo_name: bio-ontology-research-group/deeppheno

max_forks_repo_name: bio-ontology-research-group/deeppheno


text:
import datetime
import os
import time

import numpy as np

import getml.data as data
import getml.database as database
import getml.engine as engine

# -----------------------------------------------------------------------------
# Set up the MySQL connection.

database.connect_mysql(
    host="localhost",
    port=3306,
    dbname="mydb",
    user="myuser",
    password="mypassword",
    time_formats=['%Y/%m/%d']
)

# -----------------------------------------------------------------------------
# Set up folders - you need to insert folders on your computer

# The folder that contains expd151.csv
RAW_DATA_FOLDER = os.getenv("HOME") + "/Downloads/diary15/"

# -----------------------------------------------------------------------------

engine.set_project("CE")

# -----------------------------------------------------------------------------
# Begin timing

begin = time.time()

# #############################################################################
# Load data.

# -----------------------------------------------------------------------------
# Load EXPD

expd_fnames = [
    RAW_DATA_FOLDER + "expd151.csv",
    RAW_DATA_FOLDER + "expd152.csv",
    RAW_DATA_FOLDER + "expd153.csv",
    RAW_DATA_FOLDER + "expd154.csv"
]

query = database.sniff_csv("EXPD_RAW", expd_fnames)

print(query)

# The sniffer will interpret UCC and NEWID
# as DOUBLE columns. But we want them
# to be treated as TEXT.
database.execute("""
DROP TABLE IF EXISTS EXPD_RAW;

CREATE TABLE EXPD_RAW(
    NEWID    TEXT,
    ALLOC    INT,
    COST     DOUBLE,
    GIFT     INT,
    PUB_FLAG INT,
    UCC      TEXT,
    EXPNSQDY TEXT,
    EXPN_QDY TEXT,
    EXPNWKDY TEXT,
    EXPN_KDY TEXT,
    EXPNMO   TEXT,
    EXPNMO_  TEXT,
    EXPNYR   TEXT,
    EXPNYR_  TEXT);
""")

database.read_csv("EXPD_RAW", expd_fnames)

# -----------------------------------------------------------------------------
# Load FMLD

# FMLD's original row size is too large for MySQL, so we have to use our own data frames
# to load the data.

fmld_fnames = [
    RAW_DATA_FOLDER + "fmld151.csv",
    RAW_DATA_FOLDER + "fmld152.csv",
    RAW_DATA_FOLDER + "fmld153.csv",
    RAW_DATA_FOLDER + "fmld154.csv"
]

FMLD_NUMERICAL = [
    "INC_RANK",
    "INC_RNK1",
    "INC_RNK2",
    "INC_RNK3",
    "INC_RNK4",
    "INC_RNK5",
    "INC_RNKM"
]

FMLD_JOIN_KEYS = [
    "NEWID"
]

fmld_roles = {
    "join_key": FMLD_JOIN_KEYS,
    "numerical": FMLD_NUMERICAL
}

df_fmld = data.DataFrame.from_csv(
    fnames=fmld_fnames,
    name="FMLD",
    roles=fmld_roles,
    ignore=True
)

df_fmld.to_db("FMLD")

# -----------------------------------------------------------------------------
# Load MEMD

memd_fnames = [
    RAW_DATA_FOLDER + "memd151.csv",
    RAW_DATA_FOLDER + "memd152.csv",
    RAW_DATA_FOLDER + "memd153.csv",
    RAW_DATA_FOLDER + "memd154.csv"
]

query = database.sniff_csv("MEMD_RAW", memd_fnames)

print(query)

database.execute("""
DROP TABLE IF EXISTS MEMD_RAW;

CREATE TABLE MEMD_RAW(
    OCCULIST TEXT,
    HRSPERWK TEXT,
    WKS_WRKD TEXT,
    EMPLTYPE TEXT,
    MARITAL  INT,
    HISPANIC TEXT,
    WHYNOWRK TEXT,
    MEMBRACE INT,
    SEX      INT,
    HRSP_RWK TEXT,
    WKS__RKD TEXT,
    EMPL_YPE TEXT,
    HISP_NIC TEXT,
    WHYN_WRK TEXT,
    OCCU_IST TEXT,
    NEWID    TEXT,
    AGE      INT,
    AGE_     TEXT,
    ANGVX    TEXT,
    ANGVX_   TEXT,
    ANPVTX   TEXT,
    ANPVTX_  TEXT,
    ANRRX    TEXT,
    ANRRX_   TEXT,
    CU_CODE1 INT,
    EDUCA    TEXT,
    EDUCA_   TEXT,
    GROSPAYX TEXT,
    GROS_AYX TEXT,
    GVX      TEXT,
    GVX_     TEXT,
    IRAX     TEXT,
    IRAX_    TEXT,
    JSSDEDX  TEXT,
    JSSDEDX_ TEXT,
    MEMBNO   INT,
    PVTX     TEXT,
    PVTX_    TEXT,
    RRX      TEXT,
    RRX_     TEXT,
    SCHLNCHQ TEXT,
    SCHL_CHQ TEXT,
    SCHLNCHX TEXT,
    SCHL_CHX TEXT,
    SLFEMPSS TEXT,
    SLFE_PSS TEXT,
    SS_RRX   TEXT,
    SS_RRX_  TEXT,
    SUPPX    TEXT,
    SUPPX_   TEXT,
    US_SUPP  TEXT,
    US_SUPP_ TEXT,
    WAGEX    TEXT,
    WAGEX_   TEXT,
    SS_RRQ   TEXT,
    SS_RRQ_  TEXT,
    SOCRRX   TEXT,
    SOCRRX_  TEXT,
    ARM_FORC TEXT,
    ARM__ORC TEXT,
    IN_COLL  TEXT,
    IN_COLL_ TEXT,
    MEDICARE TEXT,
    MEDI_ARE TEXT,
    PAYPERD  TEXT,
    PAYPERD_ TEXT,
    HORIGIN  INT,
    RC_WHITE TEXT,
    RC_W_ITE TEXT,
    RC_BLACK TEXT,
    RC_B_ACK TEXT,
    RC_NATAM TEXT,
    RC_N_TAM TEXT,
    RC_ASIAN TEXT,
    RC_A_IAN TEXT,
    RC_PACIL TEXT,
    RC_P_CIL TEXT,
    RC_OTHER TEXT,
    RC_O_HER TEXT,
    RC_DK    TEXT,
    RC_DK_   TEXT,
    ANGVXM   TEXT,
    ANGVXM_  TEXT,
    ANPVTXM  TEXT,
    ANPVTXM_ TEXT,
    ANRRXM   TEXT,
    ANRRXM_  TEXT,
    JSSDEDXM TEXT,
    JSSD_DXM TEXT,
    JSSDEDX1 TEXT,
    JSSDEDX2 TEXT,
    JSSDEDX3 TEXT,
    JSSDEDX4 TEXT,
    JSSDEDX5 TEXT,
    SLFEMPSM TEXT,
    SLFE_PSM TEXT,
    SLFEMPS1 TEXT,
    SLFEMPS2 TEXT,
    SLFEMPS3 TEXT,
    SLFEMPS4 TEXT,
    SLFEMPS5 TEXT,
    SOCRRXM  TEXT,
    SOCRRXM_ TEXT,
    SOCRRX1  TEXT,
    SOCRRX2  TEXT,
    SOCRRX3  TEXT,
    SOCRRX4  TEXT,
    SOCRRX5  TEXT,
    SS_RRXM  TEXT,
    SS_RRXM_ TEXT,
    SS_RRX1  TEXT,
    SS_RRX2  TEXT,
    SS_RRX3  TEXT,
    SS_RRX4  TEXT,
    SS_RRX5  TEXT,
    SS_RRXI  TEXT,
    SUPPXM   TEXT,
    SUPPXM_  TEXT,
    SUPPX1   TEXT,
    SUPPX2   TEXT,
    SUPPX3   TEXT,
    SUPPX4   TEXT,
    SUPPX5   TEXT,
    SUPPXI   TEXT,
    WAGEXM   TEXT,
    WAGEXM_  TEXT,
    WAGEX1   TEXT,
    WAGEX2   TEXT,
    WAGEX3   TEXT,
    WAGEX4   TEXT,
    WAGEX5   TEXT,
    WAGEXI   TEXT,
    SS_RRB   TEXT,
    SS_RRB_  TEXT,
    SS_RRBX  TEXT,
    SS_RRBX_ TEXT,
    SUPPB    TEXT,
    SUPPB_   TEXT,
    SUPPBX   TEXT,
    SUPPBX_  TEXT,
    WAGEB    TEXT,
    WAGEB_   TEXT,
    WAGEBX   TEXT,
    WAGEBX_  TEXT,
    ASIAN    TEXT,
    ASIAN_   TEXT,
    OCCUEARN TEXT,
    PAYSTUB  TEXT,
    PAYSTUB_ TEXT,
    SEMPFRM  TEXT,
    SEMPFRM_ TEXT,
    SEMPFRMX TEXT,
    SEMP_RMX TEXT,
    SMPFRMB  TEXT,
    SMPFRMB_ TEXT,
    SMPFRMBX TEXT,
    SMPF_MBX TEXT,
    SEMPFRM1 TEXT,
    SEMPFRM2 TEXT,
    SEMPFRM3 TEXT,
    SEMPFRM4 TEXT,
    SEMPFRM5 TEXT,
    SEMPFRMI TEXT,
    SEMPFRMM TEXT,
    SEMP_RMM TEXT,
    SOCSRRET TEXT,
    SOCS_RET TEXT,
    WKSTATUS TEXT);
""")

database.read_csv("MEMD_RAW", memd_fnames)

# -----------------------------------------------------------------------------
# Preprocess EXPD.

database.execute("""
    DROP TABLE IF EXISTS EXPD;

    CREATE TABLE EXPD AS
    SELECT CASE WHEN GIFT=2 THEN 0 ELSE 1 END AS TARGET,
           CONCAT(EXPNYR, "/", EXPNMO, "/", "01") AS TIME_STAMP,
           NEWID,
           EXPNYR,
           CAST(EXPNMO AS UNSIGNED) AS EXPNMO,
           COST,
           SUBSTR(UCC, 1, 1) AS UCC1,
           SUBSTR(UCC, 1, 2) AS UCC2,
           SUBSTR(UCC, 1, 3) AS UCC3,
           SUBSTR(UCC, 1, 4) AS UCC4,
           SUBSTR(UCC, 1, 5) AS UCC5,
           SUBSTR(UCC, 1, 6) AS UCC
    FROM EXPD_RAW
    WHERE EXPNMO != '';
""")

# -----------------------------------------------------------------------------
# Preprocess MEMD.

database.execute("""
    DROP TABLE IF EXISTS MEMD;

    CREATE TABLE MEMD AS
    SELECT MARITAL,
           SEX,
           EMPLTYPE,
           HISPANIC,
           OCCULIST,
           WHYNOWRK,
           EDUCA,
           MEDICARE,
           PAYPERD,
           RC_WHITE,
           RC_BLACK,
           RC_ASIAN,
           RC_OTHER,
           WKSTATUS,
           AGE,
           WAGEX,
           NEWID,
           '2015/01/01' AS TIME_STAMP
    FROM MEMD_RAW
""")

# -----------------------------------------------------------------------------
# Make POPULATION TABLE

database.execute("""
    CREATE INDEX EXPD_INDEX ON EXPD(NEWID(10));
    CREATE INDEX FMLD_INDEX ON FMLD(NEWID(10));

    DROP TABLE IF EXISTS POPULATION_ALL;

    CREATE TABLE POPULATION_ALL AS
    SELECT t1.*,
           t2.INC_RANK,
           t2.INC_RNK1,
           t2.INC_RNK2,
           t2.INC_RNK3,
           t2.INC_RNK4,
           t2.INC_RNK5,
           t2.INC_RNKM
    FROM EXPD t1
    LEFT JOIN FMLD t2
    ON t1.NEWID = t2.NEWID;
""")

# -----------------------------------------------------------------------------
# Load EXPD into the engine.

EXPD_CATEGORICAL = [
    "UCC",
    "UCC1",
    "UCC2",
    "UCC3",
    "UCC4",
    "UCC5"
]

EXPD_JOIN_KEYS = [
    "NEWID"
]

EXPD_NUMERICAL = [
    "COST",
    "EXPNYR",
    "EXPNMO"
]

EXPD_TARGETS = [
    "TARGET"
]

EXPD_TIME_STAMPS = [
    "TIME_STAMP"
]

expd_roles = {
    "join_key": EXPD_JOIN_KEYS,
    "time_stamp": EXPD_TIME_STAMPS,
    "categorical": EXPD_CATEGORICAL,
    "numerical": EXPD_NUMERICAL,
    "target": EXPD_TARGETS
}

df_expd = data.DataFrame.from_db(
    table_name="EXPD",
    name="EXPD",
    roles=expd_roles,
    ignore=True)

df_expd.set_unit("UCC1", "UCC1")
df_expd.set_unit("UCC2", "UCC2")
df_expd.set_unit("UCC3", "UCC3")
df_expd.set_unit("UCC4", "UCC4")
df_expd.set_unit("UCC5", "UCC5")
df_expd.set_unit("UCC", "UCC")
df_expd.set_unit("EXPNMO", "month")

df_expd.save()

# -----------------------------------------------------------------------------
# Load POPULATION_ALL into the engine.

FMLD_NUMERICAL = [
    "INC_RANK",
    "INC_RNK1",
    "INC_RNK2",
    "INC_RNK3",
    "INC_RNK4",
    "INC_RNK5",
    "INC_RNKM"
]

population_roles = {
    "join_key": EXPD_JOIN_KEYS,
    "time_stamp": EXPD_TIME_STAMPS,
    "categorical": EXPD_CATEGORICAL,
    "numerical": EXPD_NUMERICAL + FMLD_NUMERICAL,
    "target": EXPD_TARGETS
}

df_population_all = data.DataFrame.from_db(
    table_name="POPULATION_ALL",
    name="POPULATION_ALL",
    roles=population_roles,
    ignore=True)

df_population_all.set_unit("UCC1", "UCC1")
df_population_all.set_unit("UCC2", "UCC2")
df_population_all.set_unit("UCC3", "UCC3")
df_population_all.set_unit("UCC4", "UCC4")
df_population_all.set_unit("UCC5", "UCC5")
df_population_all.set_unit("UCC", "UCC")
df_population_all.set_unit("EXPNMO", "month")

df_population_all.save()

# -----------------------------------------------------------------------------
# Separate POPULATION_ALL into training, testing, validation set.

random = df_population_all.random()

df_population_training = df_population_all.where("POPULATION_TRAINING", random <= 0.7)

df_population_training.save()

df_population_validation = df_population_all.where("POPULATION_VALIDATION", (random <= 0.85) & (random > 0.7))

df_population_validation.save()

df_population_testing = df_population_all.where("POPULATION_TESTING", random > 0.85)

df_population_testing.save()

# -----------------------------------------------------------------------------
# Load MEMD

MEMD_CATEGORICAL = [
    "MARITAL",
    "SEX",
    "EMPLTYPE",
    "HISPANIC",
    "OCCULIST",
    "WHYNOWRK",
    "EDUCA",
    "MEDICARE",
    "PAYPERD",
    "RC_WHITE",
    "RC_BLACK",
    "RC_ASIAN",
    "RC_OTHER",
    "WKSTATUS"
]

MEMD_NUMERICAL = [
    "AGE",
    "WAGEX",
]

MEMD_JOIN_KEYS = [
    "NEWID"
]

MEMD_TIME_STAMPS = [
    "TIME_STAMP"
]

memd_roles = {
    "join_key": MEMD_JOIN_KEYS,
    "time_stamp": MEMD_TIME_STAMPS,
    "categorical": MEMD_CATEGORICAL,
    "numerical": MEMD_NUMERICAL
}

df_memd = data.DataFrame.from_db(
    table_name="MEMD",
    name="MEMD",
    roles=memd_roles,
    ignore=True)

df_memd.save()

# -----------------------------------------------------------------------------
# Print time taken

end = time.time()

print("Time taken: " + str(end - begin) + " seconds.")

max_stars_repo_name: srnnkls/getml-examples

max_issues_repo_name: srnnkls/getml-examples

max_forks_repo_name: srnnkls/getml-examples


text:
# Common imports
import numpy as np
import os
from backend import import_excel, export_excel
# To plot pretty figures
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.style as style
# style.use('bmh')
from mpl_toolkits.mplot3d import Axes3D
mpl.rc('axes', labelsize=14)
mpl.rc('xtick', labelsize=12)
mpl.rc('ytick', labelsize=12)

import pandas as pd

import tensorflow as tf
import keras
import random

import sys
sys.path.append("..")

import dataset, network
import WGAN_Model

# Load data
# 2 n_features sets "sinus", "circle", "multi", "moons", "heter"
# 3 n_features sets "helix", "3d"

scenario = "moons"
n_instance = 1000 # number of generated points
n_features = 2

if scenario in ("3d", "helix") :
    X_train, y_train, X_test, y_test, X_valid, y_valid = dataset.get_dataset(n_instance, scenario)
else:
    X_train, y_train, X_test, y_test, X_valid, y_valid = dataset.get_dataset(n_instance, scenario)

os.system('mkdir Dataset')
os.system('mkdir GANS')
os.system('mkdir GANS/Models')
os.system('mkdir GANS/Losses')
os.system('mkdir GANS/Random_test')
export_excel(X_train, 'Dataset/X_train')
export_excel(y_train, 'Dataset/y_train')
# print(X_train.shape,y_train.shape)
X_train = import_excel('Dataset/X_train')
y_train = import_excel('Dataset/y_train')
print('made dataset')
# Preprocessing
vars = np.zeros((6,864))
j = 0
for i in range(6):
    for i2 in range(4):
        for i3 in range(3):
            for i4 in range(2):
                for i5 in range(3):
                    for i6 in range(2):
                        vars[0,j]=i+2
                        vars[1,j]=i2
                        vars[2,j]=i3
                        vars[3,j]=i4
                        vars[4,j]=i5
                        vars[5,j]=i6
                        j = j +1
j = 0#int(sys.argv[1])-1
print(vars[:,j])
n_features = 2
n_var =int(vars[0,j])
latent_spaces = [3,10,50,100]
latent_space = 10#int(latent_spaces[int(vars[1,j])])
batchs = [10,100,1000]
BATCH_SIZE = 100#int(batchs[int(vars[2,j])])
scales = ['-1-1','0-1']
scaled = '-1-1'#scales[int(vars[3,j])]
epochs = 501   #[1000,10000,10000]
# epoch = int(epochs[int(vars[4,j])])
bias = [True,False]
use_bias = True#(bias[int(vars[5,j])])

wgan = WGAN_Model.WGAN(n_features,latent_space,BATCH_SIZE,n_var,use_bias)
train_dataset, scaler, X_train_scaled = wgan.preproc(X_train, y_train, scaled)
hist = wgan.train(train_dataset, epochs, scaler, scaled, X_train, y_train)
wgan.generator.save('GANS/Models/GAN_'+str(j))
# plot loss
print('Loss: ')
fig, ax = plt.subplots(1,1, figsize=[10,5])
ax.plot(hist)
ax.legend(['loss_gen', 'loss_disc'])
#ax.set_yscale('log')
ax.grid()
plt.tight_layout()
plt.savefig('GANS/Losses/GANS_loss'+str(j)+'.png')
generator = keras.models.load_model('GANS/Models/GAN_'+str(j))
plt.close()
# latent_values = tf.random.normal([1000, latent_space], mean=0.0, stddev=0.1)
# predicted_values = wgan.generator.predict(latent_values)
# if scaled == '-1-1':
#     predicted_values[:,:]=(predicted_values[:,:]+1)/2
#     predicted_values = scaler.inverse_transform(predicted_values)
# elif scaled =='0-1':
#     predicted_values = scaler.inverse_transform(predicted_values)
# plt.plot(X_train,y_train,'o')
# plt.plot(predicted_values[:,0],predicted_values[:,1],'o')
# plt.savefig('GANS/Random_test/GANS_test'+str(j)+'.png')
print('stop')
# Prediction
# define these for desired prediction
x_input = [-1, 0, 0.5, 1.5]
n_points = 80
y_min = -0.75
y_max = 1

# produces an input of fixed x coordinates with random y values
predict1 = np.full((n_points//4, 2), x_input[0])
predict2 = np.full((n_points//4, 2), x_input[1])
predict3 = np.full((n_points//4, 2), x_input[2])
predict4 = np.full((n_points//4, 2), x_input[3])
predictthis = np.concatenate((predict1, predict2, predict3, predict4))

for n in range(n_points):
    predictthis[n,1] = 0

predictthis_scaled = scaler.transform(predictthis)
# input_test = predictthis_scaled.reshape(-1, n_features).astype('float32')
X_generated = wgan.predict(predictthis_scaled, scaler)

# test_dataset, scaler1, X_test_scaled = wgan.preproc(X_test, y_test, scaled)
#
# X_generated = wgan.predict(X_test_scaled, scaler)

plt.clf()
if scenario in ("3d", "helix"):
    ax = plt.subplot(projection='3d')

    ax.scatter(X_generated[:,0], X_generated[:,1], X_generated[:,2], c='b', label='Generated Data')
    ax.scatter(X_train[:,0], X_train[:,1], y_train, c='r')

    ax.set_zlabel('Z')
    ax.set_ylabel('Y')
    ax.set_xlabel('X')

    plt.legend(loc='best')
    plt.tight_layout()
    plt.savefig("pred_moons.png")

else:
    plt.title("Prediction at x = -1, 0, 0.5, 1.5")
    plt.scatter(X_train, y_train, label="Training data")
    #plt.scatter(X_test, y_test, label="Test data")
    #plt.scatter(predictthis[:,0], predictthis[:,1], label="Sample data", c="pink")
    plt.scatter(X_generated[:,0], X_generated[:,1], label="Fixed Input Prediction")
    plt.legend(loc='upper right')
    plt.tight_layout()
    plt.xlabel("x")
    plt.ylabel("y")
    plt.savefig("pred_moons.png")

max_stars_repo_name: EllyessB/WGAN-Regression

max_issues_repo_name: EllyessB/WGAN-Regression

max_forks_repo_name: EllyessB/WGAN-Regression