text:
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
import numpy as np

""" Helper class """
class Helpers(object):
    """
    A Helper Module
    You can:
        1. plot_decision_regions
        2. generate_random_2d_data
    """

    def __init__(self):
        """ Default initializer for Helpers class"""

    def plot_decision_regions(self, _data_, _target_, classifier, resolution=0.02):
        """ Plot decision boundary mesh """
        # setup marker generator and color map
        markers = ('s', 'x', 'o', '^', 'v')
        colors = ('red', 'blue', 'lightgreen', 'gray', 'cyan')
        cmap = ListedColormap(colors[:len(np.unique(_target_))])

        # plot the decision surface
        x1_min, x1_max = _data_[:, 0].min() - 1, _data_[:, 0].max() + 1
        x2_min, x2_max = _data_[:, 1].min() - 1, _data_[:, 1].max() + 1
        xx1, xx2 = np.meshgrid(np.arange(x1_min, x1_max, resolution),
                               np.arange(x2_min, x2_max, resolution))
        predicted_class = classifier.predict(
            np.array([xx1.ravel(), xx2.ravel()]).T)
        predicted_class = predicted_class.reshape(xx1.shape)
        plt.contourf(xx1, xx2, predicted_class, alpha=0.4, cmap=cmap)
        plt.xlim(xx1.min(), xx1.max())
        plt.ylim(xx2.min(), xx2.max())

        # plot class samples
        for idx, classy_ in enumerate(np.unique(_target_)):
            plt.scatter(x=_data_[_target_ == classy_, 0], y=_data_[_target_ == classy_, 1],
                        alpha=0.8, c=cmap(idx),
                        marker=markers[idx], label=classy_)

    def generate_random_2d_data(self, number_of_samples=200):
        """
        Generates a 2D linearly separable dataset with n samples.
        The third element of the sample is the label
        """
        x_b = (np.random.rand(number_of_samples)*2-1)/2-0.5
        y_b = (np.random.rand(number_of_samples)*2-1)/2+0.5
        x_r = (np.random.rand(number_of_samples)*2-1)/2+0.5
        y_r = (np.random.rand(number_of_samples)*2-1)/2-0.5

        inputs = []
        labels = []
        for i, _ in enumerate(x_b):
            inputs.append([x_b[i], y_b[i]])
            labels.append(1)
            inputs.append([x_r[i], y_r[i]])
            labels.append(-1)

        return inputs, labels

max_stars_repo_name: nitish-tripathi/Simplery

max_issues_repo_name: nitish-tripathi/Simplery

max_forks_repo_name: nitish-tripathi/Simplery


text:
import torch
from common import knn, rigid_transform_3d
from utils.SE3 import transform
import numpy as np

class Matcher():
    def __init__(self,
                 inlier_threshold=0.10,
                 num_node='all',
                 use_mutual=True,
                 d_thre=0.1,
                 num_iterations=10,
                 ratio=0.2,
                 nms_radius=0.1,
                 max_points=8000,
                 k1=30,
                 k2=20,
                 select_scene=None,
                 ):
        self.inlier_threshold = inlier_threshold
        self.num_node = num_node
        self.use_mutual = use_mutual
        self.d_thre = d_thre
        self.num_iterations = num_iterations  # maximum iteration of power iteration algorithm
        self.ratio = ratio # the maximum ratio of seeds.
        self.max_points = max_points
        self.nms_radius = nms_radius
        self.k1 = k1
        self.k2 = k2

    def pick_seeds(self, dists, scores, R, max_num):
        """
        Select seeding points using Non Maximum Suppression. (here we only support bs=1)
        Input:
            - dists:       [bs, num_corr, num_corr] src keypoints distance matrix
            - scores:      [bs, num_corr]     initial confidence of each correspondence
            - R:           float              radius of nms
            - max_num:     int                maximum number of returned seeds
        Output:
            - picked_seeds: [bs, num_seeds]   the index to the seeding correspondences
        """
        assert scores.shape[0] == 1

        # parallel Non Maximum Suppression (more efficient)
        score_relation = scores.T >= scores  # [num_corr, num_corr], save the relation of leading_eig
        # score_relation[dists[0] >= R] = 1  # mask out the non-neighborhood node
        score_relation = score_relation.bool() | (dists[0] >= R).bool()
        is_local_max = score_relation.min(-1)[0].float()

        score_local_max = scores * is_local_max
        sorted_score = torch.argsort(score_local_max, dim=1, descending=True)

        # max_num = scores.shape[1]

        return_idx = sorted_score[:, 0: max_num].detach()

        return return_idx

    def cal_seed_trans(self, seeds, SC2_measure, src_keypts, tgt_keypts):
        """
        Calculate the transformation for each seeding correspondences.
        Input:
            - seeds:         [bs, num_seeds]              the index to the seeding correspondence
            - SC2_measure: [bs, num_corr, num_channels]
            - src_keypts:    [bs, num_corr, 3]
            - tgt_keypts:    [bs, num_corr, 3]
        Output: leading eigenvector
            - final_trans:       [bs, 4, 4]             best transformation matrix (after post refinement) for each batch.
        """
        bs, num_corr, num_channels = SC2_measure.shape[0], SC2_measure.shape[1], SC2_measure.shape[2]
        k1 = self.k1
        k2 = self.k2

        if k1 > num_channels:
            k1 = 4
            k2 = 4

        #################################
        # The first stage consensus set sampling
        # Finding the k1 nearest neighbors around each seed
        #################################
        sorted_score = torch.argsort(SC2_measure, dim=2, descending=True)
        knn_idx = sorted_score[:, :, 0: k1]
        sorted_value, _ = torch.sort(SC2_measure, dim=2, descending=True)
        idx_tmp = knn_idx.contiguous().view([bs, -1])
        idx_tmp = idx_tmp[:, :, None]
        idx_tmp = idx_tmp.expand(-1, -1, 3)

        #################################
        # construct the local SC2 measure of each consensus subset obtained in the first stage.
        #################################
        src_knn = src_keypts.gather(dim=1, index=idx_tmp).view([bs, -1, k1, 3])  # [bs, num_seeds, k, 3]
        tgt_knn = tgt_keypts.gather(dim=1, index=idx_tmp).view([bs, -1, k1, 3])
        src_dist = ((src_knn[:, :, :, None, :] - src_knn[:, :, None, :, :]) ** 2).sum(-1) ** 0.5
        tgt_dist = ((tgt_knn[:, :, :, None, :] - tgt_knn[:, :, None, :, :]) ** 2).sum(-1) ** 0.5
        cross_dist = torch.abs(src_dist - tgt_dist)
        local_hard_SC_measure = (cross_dist < self.d_thre).float()
        local_SC2_measure = torch.matmul(local_hard_SC_measure[:, :, :1, :], local_hard_SC_measure)

        #################################
        # perform second stage consensus set sampling
        #################################
        sorted_score = torch.argsort(local_SC2_measure, dim=3, descending=True)
        knn_idx_fine = sorted_score[:, :, :, 0: k2]

        #################################
        # construct the soft SC2 matrix of the consensus set
        #################################
        num = knn_idx_fine.shape[1]
        knn_idx_fine = knn_idx_fine.contiguous().view([bs, num, -1])[:, :, :, None]
        knn_idx_fine = knn_idx_fine.expand(-1, -1, -1, 3)
        src_knn_fine = src_knn.gather(dim=2, index=knn_idx_fine).view([bs, -1, k2, 3])  # [bs, num_seeds, k, 3]
        tgt_knn_fine = tgt_knn.gather(dim=2, index=knn_idx_fine).view([bs, -1, k2, 3])

        src_dist = ((src_knn_fine[:, :, :, None, :] - src_knn_fine[:, :, None, :, :]) ** 2).sum(-1) ** 0.5
        tgt_dist = ((tgt_knn_fine[:, :, :, None, :] - tgt_knn_fine[:, :, None, :, :]) ** 2).sum(-1) ** 0.5
        cross_dist = torch.abs(src_dist - tgt_dist)
        local_hard_measure = (cross_dist < self.d_thre * 2).float()
        local_SC2_measure = torch.matmul(local_hard_measure, local_hard_measure) / k2
        local_SC_measure = torch.clamp(1 - cross_dist ** 2 / self.d_thre ** 2, min=0)
        # local_SC2_measure = local_SC_measure * local_SC2_measure
        local_SC2_measure = local_SC_measure
        local_SC2_measure = local_SC2_measure.view([-1, k2, k2])

        #################################
        # Power iteratation to get the inlier probability
        #################################
        local_SC2_measure[:, torch.arange(local_SC2_measure.shape[1]), torch.arange(local_SC2_measure.shape[1])] = 0
        total_weight = self.cal_leading_eigenvector(local_SC2_measure, method='power')
        total_weight = total_weight.view([bs, -1, k2])
        total_weight = total_weight / (torch.sum(total_weight, dim=-1, keepdim=True) + 1e-6)

        #################################
        # calculate the transformation by weighted least-squares for each subsets in parallel
        #################################
        total_weight = total_weight.view([-1, k2])
        src_knn = src_knn_fine
        tgt_knn = tgt_knn_fine
        src_knn, tgt_knn = src_knn.view([-1, k2, 3]), tgt_knn.view([-1, k2, 3])

        #################################
        # compute the rigid transformation for each seed by the weighted SVD
        #################################
        seedwise_trans = rigid_transform_3d(src_knn, tgt_knn, total_weight)
        seedwise_trans = seedwise_trans.view([bs, -1, 4, 4])

        #################################
        # calculate the inlier number for each hypothesis, and find the best transformation for each point cloud pair
        #################################
        pred_position = torch.einsum('bsnm,bmk->bsnk', seedwise_trans[:, :, :3, :3],
                                     src_keypts.permute(0, 2, 1)) + seedwise_trans[:, :, :3,
                                                                    3:4]  # [bs, num_seeds, num_corr, 3]
        #################################
        # calculate the inlier number for each hypothesis, and find the best transformation for each point cloud pair
        #################################
        pred_position = pred_position.permute(0, 1, 3, 2)
        L2_dis = torch.norm(pred_position - tgt_keypts[:, None, :, :], dim=-1)  # [bs, num_seeds, num_corr]
        seedwise_fitness = torch.sum((L2_dis < self.inlier_threshold).float(), dim=-1)  # [bs, num_seeds]
        batch_best_guess = seedwise_fitness.argmax(dim=1)
        best_guess_ratio = seedwise_fitness[0, batch_best_guess]
        final_trans = seedwise_trans.gather(dim=1,index=batch_best_guess[:, None, None, None].expand(-1, -1, 4, 4)).squeeze(1)

        return final_trans

    def cal_leading_eigenvector(self, M, method='power'):
        """
        Calculate the leading eigenvector using power iteration algorithm or torch.symeig
        Input:
            - M:      [bs, num_corr, num_corr] the compatibility matrix
            - method: select different method for calculating the learding eigenvector.
        Output:
            - solution: [bs, num_corr] leading eigenvector
        """
        if method == 'power':
            # power iteration algorithm
            leading_eig = torch.ones_like(M[:, :, 0:1])
            leading_eig_last = leading_eig
            for i in range(self.num_iterations):
                leading_eig = torch.bmm(M, leading_eig)
                leading_eig = leading_eig / (torch.norm(leading_eig, dim=1, keepdim=True) + 1e-6)
                if torch.allclose(leading_eig, leading_eig_last):
                    break
                leading_eig_last = leading_eig
            leading_eig = leading_eig.squeeze(-1)
            return leading_eig
        elif method == 'eig':  # cause NaN during back-prop
            e, v = torch.symeig(M, eigenvectors=True)
            leading_eig = v[:, :, -1]
            return leading_eig
        else:
            exit(-1)

    def cal_confidence(self, M, leading_eig, method='eig_value'):
        """
        Calculate the confidence of the spectral matching solution based on spectral analysis.
        Input:
            - M:          [bs, num_corr, num_corr] the compatibility matrix
            - leading_eig [bs, num_corr]           the leading eigenvector of matrix M
        Output:
            - confidence
        """
        if method == 'eig_value':
            # max eigenvalue as the confidence (Rayleigh quotient)
            max_eig_value = (leading_eig[:, None, :] @ M @ leading_eig[:, :, None]) / (
                        leading_eig[:, None, :] @ leading_eig[:, :, None])
            confidence = max_eig_value.squeeze(-1)
            return confidence
        elif method == 'eig_value_ratio':
            # max eigenvalue / second max eigenvalue as the confidence
            max_eig_value = (leading_eig[:, None, :] @ M @ leading_eig[:, :, None]) / (
                        leading_eig[:, None, :] @ leading_eig[:, :, None])
            # compute the second largest eigen-value
            B = M - max_eig_value * leading_eig[:, :, None] @ leading_eig[:, None, :]
            solution = torch.ones_like(B[:, :, 0:1])
            for i in range(self.num_iterations):
                solution = torch.bmm(B, solution)
                solution = solution / (torch.norm(solution, dim=1, keepdim=True) + 1e-6)
            solution = solution.squeeze(-1)
            second_eig = solution
            second_eig_value = (second_eig[:, None, :] @ B @ second_eig[:, :, None]) / (
                        second_eig[:, None, :] @ second_eig[:, :, None])
            confidence = max_eig_value / second_eig_value
            return confidence
        elif method == 'xMx':
            # max xMx as the confidence (x is the binary solution)
            # rank = torch.argsort(leading_eig, dim=1, descending=True)[:, 0:int(M.shape[1]*self.ratio)]
            # binary_sol = torch.zeros_like(leading_eig)
            # binary_sol[0, rank[0]] = 1
            confidence = leading_eig[:, None, :] @ M @ leading_eig[:, :, None]
            confidence = confidence.squeeze(-1) / M.shape[1]
            return confidence

    def post_refinement(self, initial_trans, src_keypts, tgt_keypts, it_num, weights=None):
        """
        Perform post refinement using the initial transformation matrix, only adopted during testing.
        Input
            - initial_trans: [bs, 4, 4]
            - src_keypts:    [bs, num_corr, 3]
            - tgt_keypts:    [bs, num_corr, 3]
            - weights:       [bs, num_corr]
        Output:
            - final_trans:   [bs, 4, 4]
        """
        assert initial_trans.shape[0] == 1
        inlier_threshold = 1.2

        # inlier_threshold_list = [self.inlier_threshold] * it_num

        if self.inlier_threshold == 0.10:  # for 3DMatch
            inlier_threshold_list = [0.10] * it_num
        else:  # for KITTI
            inlier_threshold_list = [1.2] * it_num

        previous_inlier_num = 0
        for inlier_threshold in inlier_threshold_list:
            warped_src_keypts = transform(src_keypts, initial_trans)

            L2_dis = torch.norm(warped_src_keypts - tgt_keypts, dim=-1)
            pred_inlier = (L2_dis < inlier_threshold)[0]  # assume bs = 1
            inlier_num = torch.sum(pred_inlier)
            if abs(int(inlier_num - previous_inlier_num)) < 1:
                break
            else:
                previous_inlier_num = inlier_num
            initial_trans = rigid_transform_3d(
                A=src_keypts[:, pred_inlier, :],
                B=tgt_keypts[:, pred_inlier, :],
                ## https://link.springer.com/article/10.1007/s10589-014-9643-2
                # weights=None,
                weights=1 / (1 + (L2_dis / inlier_threshold) ** 2)[:, pred_inlier],
                # weights=((1-L2_dis/inlier_threshold)**2)[:, pred_inlier]
            )
        return initial_trans

    def match_pair(self, src_keypts, tgt_keypts, src_features, tgt_features):
        N_src = src_features.shape[1]
        N_tgt = tgt_features.shape[1]
        # use all point or sample points.
        if self.num_node == 'all':
            src_sel_ind = np.arange(N_src)
            tgt_sel_ind = np.arange(N_tgt)
        else:
            src_sel_ind = np.random.choice(N_src, self.num_node)
            tgt_sel_ind = np.random.choice(N_tgt, self.num_node)
        src_desc = src_features[:, src_sel_ind, :]
        tgt_desc = tgt_features[:, tgt_sel_ind, :]
        src_keypts = src_keypts[:, src_sel_ind, :]
        tgt_keypts = tgt_keypts[:, tgt_sel_ind, :]

        # match points in feature space.
        distance = torch.sqrt(2 - 2 * (src_desc[0] @ tgt_desc[0].T) + 1e-6)
        distance = distance.unsqueeze(0)
        source_idx = torch.argmin(distance[0], dim=1)
        corr = torch.cat([torch.arange(source_idx.shape[0])[:, None].cuda(), source_idx[:, None]], dim=-1)

        # generate correspondences
        src_keypts_corr = src_keypts[:, corr[:, 0]]
        tgt_keypts_corr = tgt_keypts[:, corr[:, 1]]

        return src_keypts_corr, tgt_keypts_corr

    def SC2_PCR(self, src_keypts, tgt_keypts):
        """
        Input:
            - src_keypts: [bs, num_corr, 3]
            - tgt_keypts: [bs, num_corr, 3]
        Output:
            - pred_trans:   [bs, 4, 4], the predicted transformation matrix.
            - pred_labels:  [bs, num_corr], the predicted inlier/outlier label (0,1), for classification loss calculation.
        """
        bs, num_corr = src_keypts.shape[0], tgt_keypts.shape[1]

        #################################
        # downsample points
        #################################
        if num_corr > self.max_points:
            src_keypts = src_keypts[:, :self.max_points, :]
            tgt_keypts = tgt_keypts[:, :self.max_points, :]
            num_corr = self.max_points

        #################################
        # compute cross dist
        #################################
        src_dist = torch.norm((src_keypts[:, :, None, :] - src_keypts[:, None, :, :]), dim=-1)
        target_dist = torch.norm((tgt_keypts[:, :, None, :] - tgt_keypts[:, None, :, :]), dim=-1)
        cross_dist = torch.abs(src_dist - target_dist)

        #################################
        # compute first order measure
        #################################
        SC_dist_thre = self.d_thre
        SC_measure = torch.clamp(1.0 - cross_dist ** 2 / SC_dist_thre ** 2, min=0)
        hard_SC_measure = (cross_dist < SC_dist_thre).float()

        #################################
        # select reliable seed correspondences
        #################################
        confidence = self.cal_leading_eigenvector(SC_measure, method='power')
        seeds = self.pick_seeds(src_dist, confidence, R=self.nms_radius, max_num=int(num_corr * self.ratio))

        #################################
        # compute second order measure
        #################################
        SC2_dist_thre = self.d_thre / 2
        hard_SC_measure_tight = (cross_dist < SC2_dist_thre).float()
        seed_hard_SC_measure = hard_SC_measure.gather(dim=1,
                                index=seeds[:, :, None].expand(-1, -1, num_corr))
        seed_hard_SC_measure_tight = hard_SC_measure_tight.gather(dim=1,
                                index=seeds[:, :, None].expand(-1, -1, num_corr))
        SC2_measure = torch.matmul(seed_hard_SC_measure_tight, hard_SC_measure_tight) * seed_hard_SC_measure

        #################################
        # compute the seed-wise transformations and select the best one
        #################################
        final_trans = self.cal_seed_trans(seeds, SC2_measure, src_keypts, tgt_keypts)

        #################################
        # refine the result by recomputing the transformation over the whole set
        #################################
        final_trans = self.post_refinement(final_trans, src_keypts, tgt_keypts, 20)

        return final_trans

    def estimator(self, src_keypts, tgt_keypts, src_features, tgt_features):
        """
        Input:
            - src_keypts: [bs, num_corr, 3]
            - tgt_keypts: [bs, num_corr, 3]
            - src_features: [bs, num_corr, C]
            - tgt_features: [bs, num_corr, C]
        Output:
            - pred_trans:   [bs, 4, 4], the predicted transformation matrix
            - pred_trans:   [bs, num_corr], the predicted inlier/outlier label (0,1)
            - src_keypts_corr:  [bs, num_corr, 3], the source points in the matched correspondences
            - tgt_keypts_corr:  [bs, num_corr, 3], the target points in the matched correspondences
        """
        #################################
        # generate coarse correspondences
        #################################
        src_keypts_corr, tgt_keypts_corr = self.match_pair(src_keypts, tgt_keypts, src_features, tgt_features)

        #################################
        # use the proposed SC2-PCR to estimate the rigid transformation
        #################################
        pred_trans = self.SC2_PCR(src_keypts_corr, tgt_keypts_corr)

        frag1_warp = transform(src_keypts_corr, pred_trans)
        distance = torch.sum((frag1_warp - tgt_keypts_corr) ** 2, dim=-1) ** 0.5
        pred_labels = (distance < self.inlier_threshold).float()

        return pred_trans, pred_labels, src_keypts_corr, tgt_keypts_corr

ext: py

reason: import numpy


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
transformation script for converting to long-format
"""

# add to $PYTHONPATH here if necessary to access other modules
#import sys
#sys.path.append('...')

import re
from labkeyext import ScriptReader
import pandas as pd
import numpy as np
from labkey.utils import create_server_context
from labkey.query import select_rows
from labkey.exceptions import RequestError, QueryNotFoundError, ServerContextError, ServerNotFoundError

# =============================================================================
#
# =============================================================================
sr = ScriptReader("${runInfo}", raw=True)
sr.read()
data = sr.data

# import analyte table
try:
    # query server list
    server_url = "${baseServerURL}"
    # extract just the foo.bar.bat
    server = re.search("[A-Za-z]+\.[A-Za-z]+\.[A-Za-z]+", server_url)[0]
    #TODO: automate credentials
#    sessionID = "${httpSessionId}"
#    apikey = 'session|'+sessionID
    apikey = 'apikey|............'
    """
    On your server, go to the specific table you're interested in
    and then click <export>, select the <Script> tab, then select
    the <Python> radio button and <Create Script>. Copy the
    schema_name to 'schema' and query_name to 'table' below.
    The 'folder' path is automatically updated here, though
    if needed, replace below with the second arg in the
    labkey.utils.create_server_context() call from the created script text.
    """
    folder = "${containerPath}"
    schema = 'lists'
    table = 'Analytes'
    server_context = create_server_context(server,
                                           folder,
                                           context_path='labkey',
                                            api_key=apikey)
    result = select_rows(server_context=server_context,
                         schema_name=schema,
                         query_name=table,
                         timeout=10)
    atable = pd.DataFrame(result['rows'])
    analytes = set(atable['analyte'])
    units_preset = atable.set_index('analyte')['unit'].to_dict()

    """
    Optionally include this portion if reading a file on the
    server instead of performing an API query.
    """
#    #query for local file
#    atablePath = '...analytes.tsv'
#    atable = pd.read_csv(atablePath, sep='\t')
#    analytes = set(atable['analyte'].to_list())
#    units_preset = atable.set_index('analyte')['unit'].to_dict()
#except FileNotFoundError:
except QueryNotFoundError:
    analytes = False
    units_preset = False

# detect whether a second header is present for units and process
units = {}
u = data.iloc[0].values
b = []
for v in u:
#    if is_numeric_dtype(v) and not np.isnan(v):
    if 'int' in str(type(v)) or 'float' in str(type(v)):
        b.append(True)
    else:
        b.append(False)
b = np.array(b)

if any(b):
    # second row at least one numeric type and treated as data
    units = units_preset
else:
    # second row ONLY has non-numeric data and treated as a unit specifier
    c = data.columns
    for i, v in enumerate(u):
        if type(v) is not str:
            u[i] = ''  # strike out non-strings
    units = dict(zip(c,u))
    # remove units row
    data = data.drop(index=0).reset_index(drop=True)

# =============================================================================
#
# =============================================================================
# Identify columns used as id_vars.
# These are columns that are preserved during pivot
# and are also defined by the Assay design.
idcols = ['sample_id',
          'parent_id',
          'Date']

# columns required for entry
reqdcols = {'sample_id'}

# analyte columns
pivotcols = set(data.columns) - set(idcols)

# enforce analyte names from master table
if analytes:
    illegals = pivotcols - analytes
    if len(illegals) > 0:
        # illegal names present, report error
        raise Exception("{} are not valid analyte names".format(illegals))

# fill in missing optional columns with blanks (necessary to be present for melt based on idcols)
optcols = set(idcols) - reqdcols
for c in optcols:
    if c not in data.columns:
        data[c] = ""

# convert table to long format...can't sort on sample id if nonnumeric
datalong = pd.melt(data, id_vars=idcols)#.sort_values('sample_id')
# drop any rows that don't have a value
datalong = datalong.dropna(subset=['value']).reset_index(drop=True)
# rename as necessary
datalong.rename(columns={'variable':'analyte'}, inplace=True)
# add units
if units:
    datalong['unit'] = [units[a] for a in datalong['analyte']]

# save results
datalong.to_csv(sr.filePathOut, sep='\t', index=False)

max_stars_repo_name: dasievers/lkutils

max_issues_repo_name: dasievers/lkutils

max_forks_repo_name: dasievers/lkutils


text:
import json, sys, time
import numpy as np
import pandas as pd
from sklearn.metrics import pairwise_distances_argmin
from multiprocessing import Pool, cpu_count

from helper import *

import logging
from rich.logging import RichHandler
from rich.progress import track
# from sklearn.metrics.pairwise import euclidean_distances

# logging parameters
logging.disable(logging.DEBUG)
log = logging.getLogger('rich')
logging.basicConfig(level='NOTSET', format='%(message)s', datefmt='[%X]', handlers=[RichHandler()])

def predict_route(route_id):
    """
    Returns the proposed route for a given route id.
    """

    route_info = df_new_route[route_id]
    station_code = route_info['station_code']
    zone_centers_from_station = all_zone_dict_with_coord[station_code]
    stops = route_info['stops']

    for stop in stops:
        if stops[stop]['type'] == 'Station':
            station_name = stop

    all_zone_ids = all_zone_ids_dict[route_id]
    zone_ids_list = list(set(all_zone_ids))  # unique zone ids
    C = df_new_travel[route_id].dropna()  # reading travel times

    # getting set of nodes for zone sequence prediction
    stop_zone_map = {}
    stop_zone_map[station_name] = station_code  # add station as a zone

    # map zone to node closest to zone center
    for zone_id in zone_ids_list:
        nodes_list = []
        nodes_list_coords = []
        for stop in stops:
            if stops[stop]['zone_id'] == zone_id:
                nodes_list += [stop]
                nodes_list_coords += [(stops[stop]['lat'], stops[stop]['lng'])]

        zone_coords = [zone_centers_from_station[zone_id]]  # zone center
        index_node = pairwise_distances_argmin(zone_coords, nodes_list_coords, metric='euclidean')[0]  # closest to zone center
        stop_zone_map[nodes_list[index_node]] = zone_id

    prob_matrix = prob_matrices[station_code]

    # zones = [station_code] + zone_ids_list
    # zone_centers = [zone_centers_from_station[zone_id] for zone_id in zones]

    # D_first = pd.DataFrame(euclidean_distances(zone_centers), index=zones, columns=zones)
    T = {}  # travel times between closest nodes to zone centers
    P = {}  # probabilities of moving between zone centers

    for stop_from, zone_from in stop_zone_map.items():
        T[zone_from] = {zone_to: C[stop_from][stop_to] for stop_to, zone_to in stop_zone_map.items()} ## TODO
        # D[zone_from] = {zone_to: T[zone_from][zone_to] for zone_to in stop_zone_map.values()}

        # add row and column of zeroes if zone id not in history
        if zone_from not in prob_matrix.columns:
            prob_matrix.loc[zone_from] = 0
            prob_matrix[zone_from] = 0

    for zone_from in stop_zone_map.values():
        P[zone_from] = {zone_to: prob_matrix[zone_from][zone_to] for zone_to in stop_zone_map.values()}

    T = pd.DataFrame(T)
    P = pd.DataFrame(P)
    T = T / T.max()  # TODO: Check best normalization!!

    # create cost matrix
    # omega = 0.6
    # omega_start = omega
    # omega_end = omega

    O = T.copy()
    O.iloc[:, :] = omega
    O.iloc[:, 0] = omega_start
    O.iloc[0] = omega_end

    C = O * T + (1 - O) * (1 - P)
    # C = T / (O + (1 - O) * N)

    for i in range(len(C)):  # make diagonal zero
        C.iloc[i, i] = 0.0

    # optimizing on a zone level
    zone_list = list(stop_zone_map.values())  # zone list including station
    station_zone_seq = optimize_ZL(zone_list, station_code, C)

    if not station_zone_seq:  # use Google OR if PuLP too slow
        log.warning(f'Using Google OR for route {route_id}: zone sequence')
        station_zone_seq = optimize_GoogleOR_ZL(zone_list, station_code, C)

    ordered_zones = station_zone_seq[1:]

    # optimizing on a node level
    C = df_new_travel[route_id].dropna()
    ordered_nodes = [station_name]

    for zone_idx, zone in enumerate(ordered_zones):
        # get nodes within zone
        nodes = []
        for stop in stops:
            if stops[stop]['zone_id'] == zone:
                nodes += [stop]

        num_nodes = len(nodes)

        if num_nodes == 1:
            ordered_nodes += nodes
        elif num_nodes == 2:
            node_1, node_2 = nodes[0], nodes[1]
            prev_node = ordered_nodes[-1]

            # add closest node first
            if C[prev_node][node_1] < C[prev_node][node_2]:
                ordered_nodes += [node_1, node_2]
            else:
                ordered_nodes += [node_2, node_1]
        else:
            # solve OTSP with first and last dummy nodes
            if zone_idx == len(ordered_zones) - 1:
                last_node = station_name
            else:
                next_zone_id = ordered_zones[zone_idx + 1]
                last_node = [stop for stop, zone_id in stop_zone_map.items() if zone_id == next_zone_id][0]

            first_node = ordered_nodes[-1]
            nodes = [first_node] + nodes

            route, status_NL = optimize_NL(nodes, first_node, last_node, C)  # TODO: remove first node from nodes list??

            # check if dummy node is not between first node and last node
            if status_NL != 1 or route['dummy'] != first_node or route[last_node] != 'dummy':
                log.critical(f'\nPuLP failed: route {route_id}, zone {zone}')
                log.info(f'Using Google OR for nodes {nodes}')
                route_GOR = optimize_GoogleOR_NL(nodes, first_node, last_node, C)
                ordered_nodes += route_GOR[1:]
            else:
                for _ in range(len(route) - 3):
                    ordered_nodes += [route[ordered_nodes[-1]]]

    return {'proposed': {stop: index for index, stop in enumerate(ordered_nodes)}}

if __name__ == '__main__':

    start = time.time()
    if len(sys.argv) > 1:
        data_folder = sys.argv[1]
    if len(sys.argv) == 3:
        omega = float(sys.argv[2])
        omega_start = omega
        omega_end = omega
    elif len(sys.argv) == 5:
        omega = float(sys.argv[2])
        omega_start = float(sys.argv[3])
        omega_end = float(sys.argv[4])

    # loading apply inputs
    NEW_ROUTE_DATA = f'{data_folder}/model_apply_inputs/new_route_data.json'
    NEW_TRAVEL_TIMES = f'{data_folder}/model_apply_inputs/new_travel_times.json'

    log.info(f'GET /{NEW_ROUTE_DATA}')
    df_new_route = pd.read_json(NEW_ROUTE_DATA)
    log.info(f'GET /{NEW_TRAVEL_TIMES}')
    df_new_travel = pd.read_json(NEW_TRAVEL_TIMES)
    route_ids = df_new_route.columns

    # loading build outputs
    ZONE_CENTERS = f'{data_folder}/model_build_outputs/zone_centers.json'
    log.info(f'GET /{ZONE_CENTERS}')
    df_zone_centers = pd.read_json(ZONE_CENTERS)
    stations = list(set(df_new_route.loc['station_code']))

    # read and store probability matrices
    log.info('GET probability matrices')
    prob_matrices = {}

    for station in stations:
        PROBS_STATION = f'{data_folder}/model_build_outputs/probs_from_{station}.csv'
        try:
            prob_matrices[station] = pd.read_csv(PROBS_STATION, index_col=0)
        except:
            log.warning(f'No data available for station {station}')
            prob_matrices[station] = pd.DataFrame(0, columns=[station], index=[station])

    # replace Nones by closest zone_ids and store station coordinates
    station_coords = {}
    all_zone_ids_dict = {}
    for route_id in route_ids:

        station_code = df_new_route[route_id]['station_code']
        stops_dict = df_new_route[route_id]['stops']
        all_zone_ids = [stops_dict[stop]['zone_id'] for stop in stops_dict]
        stops = list(stops_dict.keys())
        C = df_new_travel[route_id].dropna()
        all_zone_ids = remove_Nones(station_code, all_zone_ids, stops_dict, stops, C)[0]

        for i, stop in enumerate(stops):
            stop_info = stops_dict[stop]
            if stop_info['type'] == 'Dropoff':
                stop_info['zone_id'] = all_zone_ids[i]
            elif station_code not in station_coords:  # add station coords if station is new
                station_coords[station_code] = (stop_info['lat'], stop_info['lng'])

        all_zone_ids_dict[route_id] = all_zone_ids

    # for each station, check if zones are in training data, if not, add center
    log.info('Compute zone centers')
    all_zone_dict_with_coord = {}  # store dict of all (un)visited zones

    for station in stations:
        if station in df_zone_centers:
            zone_ids_centers = df_zone_centers[station].dropna()
        else:
            log.info(f'--> Station {station} has never been seen')
            zone_ids_centers = pd.Series({station: station_coords[station]})

        zone_ids = list(zone_ids_centers.index)
        zone_centers = list(zone_ids_centers.values)

        unvisited_zones = {}
        routes_from_station = df_new_route[route_ids[(df_new_route.loc['station_code'] == station)]].loc['stops']

        for route in routes_from_station:
            for stop in route.keys():
                # check if zone was not visited before
                zone_candidate = route[stop]['zone_id']
                if zone_candidate and zone_candidate not in zone_ids:
                    coords = (route[stop]['lat'], route[stop]['lng'])

                    # store all coordinates for this unvisited zone
                    if zone_candidate not in unvisited_zones:
                        unvisited_zones[zone_candidate] = [coords]
                    else:
                        unvisited_zones[zone_candidate] += [coords]

        new_zone_ids = list(unvisited_zones.keys())
        new_zone_centers = []

        # compute unvisited zone centers
        for stop_coords in unvisited_zones.values():
            mean_lat, mean_lng = np.mean(stop_coords, axis=0)
            new_zone_centers += [(mean_lat, mean_lng)]

        all_labels = zone_ids + new_zone_ids
        all_centers = zone_centers + new_zone_centers
        all_zone_dict_with_coord[station] = dict(zip(all_labels, all_centers))

    # predicting the route sequences in parallel
    log.info(f'Predict routes (omega = {omega_start, omega, omega_end})')
    with Pool(processes=cpu_count()) as p:
        results = list(track(p.imap(predict_route, route_ids), total=len(route_ids)))

    # if omega_start == omega == omega_end:
        # PROPOSED_SEQUENCES = f'{data_folder}/model_apply_outputs/proposed_sequences_omega_{omega:.2f}.json'
    # else:
    PROPOSED_SEQUENCES = f'{data_folder}/model_apply_outputs/proposed_sequences_omegas_{omega_start:.2f}-{omega:.2f}-{omega_end:.2f}.json'

    log.info(f'POST /{PROPOSED_SEQUENCES}')
    with open(PROPOSED_SEQUENCES, 'w') as file:
        json.dump(dict(zip(route_ids, results)), file)

    log.info(f'Finished in {time.time() - start} seconds')

max_stars_repo_name: donato-maragno/Amazon-LMRRC

max_issues_repo_name: donato-maragno/Amazon-LMRRC

max_forks_repo_name: donato-maragno/Amazon-LMRRC


text:
# Load pickled data
import pickle
import pandas as pd
import numpy as np
import cv2
from tensorflow.contrib.layers import flatten
import tensorflow as tf
from sklearn.utils import shuffle

"""SETTING PARAMTERS"""
MEAN = 0.0
SIGMA = 0.1
EPOCHS = 28
BATCH_SIZE = 128
LEARNING_RATE = 0.0001

''' Pre-processing pipeline
    - graysclale
    - normalize
    - reshape input data to (32,32,1)
'''

# The data is probably in order RGB
# type uint8
training_file = '../../../train.p'
validation_file= '../../../valid.p'
testing_file = '../../../test.p'

with open(training_file, mode='rb') as f:
    train = pickle.load(f)
with open(validation_file, mode='rb') as f:
    valid = pickle.load(f)
with open(testing_file, mode='rb') as f:
    test = pickle.load(f)

X_train_original, y_train_original = train['features'], train['labels']
X_valid_original, y_valid_original = valid['features'], valid['labels']
X_test_original, y_test_original = test['features'], test['labels']

'''Helper functions'''
def grayscale(input_image):
    """grayscale"""
    output = []
    for i in range(len(input_image)):
        img = cv2.cvtColor(input_image[i], cv2.COLOR_RGB2GRAY)
        output.append(img)
    return output

def normalization(input_image):
    """normalization
      Pre-defined interval [-1,1]
      from the forum :https://discussions.udacity.com/t/accuracy-is-not-going-over-75-80/314938/22
      some said that using the decimal 128.0 makes huge diffference
    """
    output = []
    for i in range(len(input_image)):
        img = np.array((input_image[i] - 128.0) / (128.0), dtype=np.float32)
        output.append(img)
    return output

def get_weights(input_shape):
    return tf.Variable(tf.truncated_normal(shape = input_shape,
                                           mean = MEAN, stddev = SIGMA))

def get_biases(length):
     return tf.Variable(tf.zeros(length))

#NOTE: number of filter is output channel
def convolution_layer(input_image,
                      filter_size,
                      input_channel,
                      number_of_filters,
                      padding_choice = 'VALID'):

    shape = [filter_size, filter_size, input_channel, number_of_filters]
    weights = get_weights(input_shape = shape)
    biases = get_biases(length = number_of_filters)
    layer = tf.nn.conv2d(input = input_image,
                         filter = weights,
                         strides = [1, 1, 1, 1],
                         padding = padding_choice) + biases
    return layer

def activation_relu(input_layer):
    return tf.nn.relu(input_layer)

def max_spooling(input_layer, padding_choice):
    return tf.nn.max_pool(value = input_layer,
                          ksize = [1, 2, 2, 1],
                          strides = [1, 2, 2, 1],
                         padding= padding_choice)

def flatten_layer(input_layer):
    return flatten(input_layer)

def fully_connected_layer(input_layer,
                          number_of_inputs,
                          number_of_outputs):

    weights = get_weights(input_shape = [number_of_inputs, number_of_outputs])
    biases = get_biases(length = number_of_outputs)
    layer = tf.matmul(input_layer, weights) + biases
    return layer

"""Pre-processing data"""

def pre_process_1st_attempt(input_image):
    gray_image = grayscale(input_image)
    normalization_image =  normalization(gray_image)
    output = np.expand_dims(normalization_image, 3)
    return output

X_train_final_1 = pre_process_1st_attempt(X_train_original)
X_valid_final_1 = pre_process_1st_attempt(X_valid_original)

"""Model design"""
def Lenet_5_first_attempt(input_image):

    # Layer 1: Convolutional. Input = 32x32x1. Output = 28x28x6.
    conv1 = convolution_layer(input_image, 5, 1, 6, 'VALID')
    conv1 = activation_relu(conv1)
    conv1 = max_spooling(conv1, 'VALID')

    # Layer 2: Convolutional. Output = 10x10x16.
    conv2 = convolution_layer(conv1, 5, 6, 16, 'VALID')
    conv2 = tf.nn.relu(conv2)
    conv2 = max_spooling(conv2, 'VALID')

    # Flatten. Input = 5x5x16. Output = 400.
    fc0   = flatten_layer(conv2)

    # Layer 3: Fully Connected. Input = 400. Output = 120.
    fc1 = fully_connected_layer(fc0, 400, 120)
    fc1 = tf.nn.relu(fc1)

    # Layer 4: Fully Connected. Input = 120. Output = 84.
    fc2 = fully_connected_layer(fc1, 120, 84)
    fc2    = tf.nn.relu(fc2)

    # Layer 5: Fully Connected. Input = 84. Output = 43
    logits = fully_connected_layer(fc2, 84, 43)

    return logits

def evaluate_first_attempt(X_data, y_data):
    """Evaluation function"""
    num_examples = len(X_data)
    total_accuracy = 0
    total_loss = 0
    sess = tf.get_default_session()
    for offset in range(0, num_examples, BATCH_SIZE):
        batch_x, batch_y = X_data[offset : offset + BATCH_SIZE], y_data[offset : offset + BATCH_SIZE]
        loss, accuracy = sess.run([loss_operation, accuracy_operation], feed_dict={x: batch_x, y: batch_y})
        total_accuracy += (accuracy * len(batch_x))
        total_loss += (loss * len(batch_x))
    return total_loss / num_examples, total_accuracy / num_examples

"""Training data"""
if __name__ == "__main__":

	x = tf.placeholder(tf.float32, (None, 32, 32, 1))
	y = tf.placeholder(tf.int32, (None))
	one_hot_y = tf.one_hot(y, 43)
	logits = Lenet_5_second_attempt(x)
	cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=one_hot_y, logits=logits)
	loss_operation = tf.reduce_mean(cross_entropy)
	optimizer = tf.train.AdamOptimizer(learning_rate = LEARNING_RATE)
	training_operation = optimizer.minimize(loss_operation)
	correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(one_hot_y, 1))
	accuracy_operation = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
	saver = tf.train.Saver()

	train_loss_history = []
	valid_loss_history = []

	#Start running tensor flow
	with tf.Session() as sess:
	training(Lenet_5_first_attempt)
	sess.run(tf.global_variables_initializer())
	num_examples = len(X_train_final_1)

	print("Training...")
	print()
	for i in range(EPOCHS):
	    X_train, y_train = shuffle(X_train_final_1, y_train_original)
	    for offset in range(0, num_examples, BATCH_SIZE):
	        end = offset + BATCH_SIZE
	        batch_x, batch_y = X_train[offset:end], y_train[offset:end]
	        sess.run(training_operation, feed_dict={x: batch_x, y: batch_y})

	    valid_loss, valid_accuracy = evaluate_first_attempt(X_valid_final_1, y_valid_original)
	    valid_loss_history.append(valid_loss)

	    train_loss, train_accuracy = evaluate_first_attempt(X_train_final_1, y_train_original)
	    train_loss_history.append(train_loss)

	    print("EPOCH {} ...".format(i+1))
	    print("Training Accuracy = {:.3f}".format(train_accuracy))
	    print("Validation Accuracy = {:.3f}".format(valid_accuracy))

	    print("Training Loss = {:.3f}".format(train_loss))
	    print("Validation Loss = {:.3f}".format(valid_loss))

	saver.save(sess, './lenet')
	print("Model saved")
	loss_plot = plt.subplot(2,1,1)
	loss_plot.set_title('Loss')
	loss_plot.plot(train_loss_history, 'r', label='Training Loss')
	loss_plot.plot(valid_loss_history, 'b', label='Validation Loss')
	loss_plot.set_xlim([0, EPOCHS])
	loss_plot.legend(loc=4)

max_stars_repo_name: tranlyvu/self-driving-car-projects

max_issues_repo_name: tranlyvu/self-driving-car-projects

max_forks_repo_name: tranlyvu/self-driving-car-projects


text:
import cv2
import numpy as np
import os
from PIL import Image, ImageSequence
import augmentors as va
from os.path import isfile, join

def image_loader(path, modality="RGB"):
    with open(path, 'rb') as f:
        with Image.open(f) as video:
            index = 1
            for frame in ImageSequence.Iterator(video):
                frames.append(frame.convert(modality))
                print (index)
                index += 1
        return frames

def get_frames_array(path):
        frames = []
        files = [f for f in os.listdir(path) if isfile(join(path, f))]

        # for sorting the file names properly
        files.sort(key = lambda x: x[5:-4])
        files.sort()

        for i in range(len(files)):
                filename = path + files[i]
                # reading each files
                img = cv2.imread(filename)
                height, width, layers = img.shape
                size = (width,height)
                print(filename)
                # inserting the frames into an image array
                frames.append(img)
        return frames, size

path = r"C:/Users/wahab/Desktop/FYP/This"
save_path = r"C:/Users/wahab/Desktop/FYP/"
dir = os.listdir(path)

for file in dir:
        print (file)

        if not os.path.exists(save_path+file):
                os.mkdir(save_path+file)

        dirs = os.listdir(path+'/'+file)
        for action in dirs:
                print (action)

        frames = []
        fps = 30
        frames, size = get_frames_array(path+'/'+file+'/')
        frames[0]
        # Horizontal Flip

        sometimes = lambda aug: va.Sometimes(1, aug) # Used to apply augmentor with 100% probability
        seq = va.Sequential([
        sometimes(va.HorizontalFlip())
        ])

        video_aug = seq(frames)
        out = cv2.VideoWriter(save_path+file+'/'+action+'_HorizontalFlip.avi',cv2.VideoWriter_fourcc(*'DIVX'), fps, size)
        for i in range(len(video_aug)):
                out.write(video_aug[i])
        out.release()

        # Vertical Flip
        frames[0]
        sometimes = lambda aug: va.Sometimes(1, aug) # Used to apply augmentor with 100% probability
        seq = va.Sequential([
        sometimes(va.VerticalFlip())
        ])

        video_aug = seq(frames)
        out = cv2.VideoWriter(save_path+file+'/'+action+'_VerticalFlip.avi',cv2.VideoWriter_fourcc(*'DIVX'), fps, size)
        for i in range(len(video_aug)):
                out.write(video_aug[i])
        out.release()

                # Below is an example to use some other augmentation techniques.

                # To add Salt effect in Video
                # frames[0]

                # sometimes = lambda aug: va.Sometimes(0.5, aug) # Used to apply augmentor with 100% probability
                # seq = va.Sequential([
                #         sometimes(va.Salt(ratio=50))
                # ])

                # video_aug = seq(frames)

                # out = cv2.VideoWriter(save_path+file+'/'+action+'_Salt.avi',cv2.VideoWriter_fourcc(*'DIVX'), fps, size)

                # for i in range(len(video_aug)):
                #         out.write(video_aug[i])
                # out.release()

                # For Random Shear in Video
                # frames[0]

                # sometimes = lambda aug: va.Sometimes(1, aug) # Used to apply augmentor with 100% probability
                # seq = va.Sequential([ # randomly rotates the video with a degree randomly choosen from [-10, 10]
                #         sometimes( va.RandomShear(x=0.3,y=0.3) )
                # ])

                # video_aug = seq(frames)

                # out = cv2.VideoWriter(save_path+file+'/'+action+'_RandomShear.avi',cv2.VideoWriter_fourcc(*'DIVX'), fps, size)

                # for i in range(len(video_aug)):
                #         out.write(video_aug[i])
                # out.release()


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Jul 26 11:35:51 2019

@author: nooteboom
"""

import numpy as np
from netCDF4 import Dataset
import calc_metrics2 as cm
import matplotlib.pylab as plt

def plotamount(nc, ty='', lim=120):
    import matplotlib.pylab as plt
    Lats = nc['temp'][:]
    sizedist = np.zeros(Lats.shape[0])
    for i in range(Lats.shape[0]):
        sizedist[i] = np.sum((Lats[i]>-200))

    sizedist = sizedist.reshape(len(nc['Lats'][:]), len(nc['Lons'][:]))
#    sizedist = (sizedist>lim)
    plt.imshow(np.flip(sizedist,0))#, vmin=0, vmax=1)
    plt.colorbar()
    plt.title('sizes of particle distributions ' + ty)
    plt.show()

sp = 6
ddeg = 1
dd = 10

minmaxlat = 100000.
maxminlat = -100000
#allresults=['monmean', 'null', '0.0','0.25', '0.5', '1.0', '2.0', '5.0']#, '0.0gm','0.25gm', '0.5gm', '1.0gm', '1.5gm', '2.0gm', '5.0gm']
#allresults= ['null',
allresults= ['monmean','0.0','0.25', '0.5', '1.0', '2.0', '5.0', '0.0gm','0.25gm','0.5gm', '1.0gm', '2.0gm', '5.0gm']#
#allresults= ['null']
#allresults= ['null','monmean']
#allresults= ['monmean']
#allresults= ['null']

dirWrite = ''
dirReadhigh =  '/projects/0/palaeo-parcels/POP/POPres/0.1degree/particlefiles/'
dirReadlow = '/projects/0/palaeo-parcels/POP/POPres/particlefiles/'
nc_target = Dataset(dirReadhigh + 'sp%d_dd10/timeseries_per_location_ddeg%d_sp%d_dd%d_tempres5_ds2.nc'%(sp,ddeg,sp,dd))

tslenmin = 10000
hid = 130#

nc = Dataset(dirReadhigh + 'sp%d_dd10/timeseries_per_location_ddeg%d_sp%d_dd%d_tempres5_ds2.nc'%(sp,ddeg,sp,dd))
#plotamount(nc, ty='ref')
minmaxlat = min(minmaxlat,np.max(nc['Lats'][:]))
maxminlat = max(maxminlat, np.min(nc['Lats'][:]))
tslen = nc['tslens'][:]
print('minlat, maxlat  '+str(np.min(nc['Lats'][:])) + '   ' + str(np.max(nc['Lats'][:])))
print(np.sum(tslen>hid)/np.float(len(tslen[tslen>0])))
tslenmin = min(tslenmin, np.min(tslen[tslen>hid]))

for i in allresults:
#for i in ['2.0', '3.0', '6.0']:
    print(i)
    if('monmean'==i):
        nc = Dataset(dirReadhigh + 'sp%d_dd10/timeseries_per_location_ddeg1_sp%d_dd10_tempresmonmean.nc'%(sp,sp))
#        plotamount(nc, ty=i)
        minmaxlat = min(minmaxlat,np.max(nc['Lats'][:]))
        maxminlat = max(maxminlat, np.min(nc['Lats'][:]))
        tslen = nc['tslens'][:]
        print('minlat, maxlat  '+str(np.min(nc['Lats'][:])) + '   ' + str(np.max(nc['Lats'][:])))
        print(np.sum(tslen>hid)/np.float(len(tslen[tslen>0])))
        tslenmin = min(tslenmin, np.min(tslen[tslen>hid]))
    elif('null' == i):
        nc = Dataset(dirReadhigh + 'sp%d_dd10/timeseries_per_location_ddeg%d_sp%d_dd%d_tempres5.nc'%(sp,ddeg,sp,dd))
#        plotamount(nc, ty=i)
        minmaxlat = min(minmaxlat,np.max(nc['Lats'][:]))
        maxminlat = max(maxminlat, np.min(nc['Lats'][:]))
        tslen = nc['tslens'][:]
        print(np.sum(tslen>hid)/np.float(len(tslen[tslen>0])))
        tslenmin = min(tslenmin, np.min(tslen[tslen>hid]))
    else:
        if(i[-2:]=='gm'):
            nc = Dataset(dirReadlow + 'sp%d_dd10/timeseries_per_location_smagorinksi_wn_gm_Cs%s_ddeg%d_sp%d_dd%d.nc'%(sp,i[:-2],ddeg,sp,dd))
        else:
            nc = Dataset(dirReadlow + 'sp%d_dd10/timeseries_per_location_smagorinksi_wn_Cs%s_ddeg%d_sp%d_dd%d.nc'%(sp,i,ddeg,sp,dd))
#        plotamount(nc, ty=i)
        minmaxlat = min(minmaxlat,np.max(nc['Lats'][:]))
        maxminlat = max(maxminlat, np.min(nc['Lats'][:]))
        vLons = nc['vLons'][:]
        vLats = nc['vLats'][:]
        tslen = nc['tslens'][:]
        print(np.sum(tslen>hid)/np.float(len(tslen[tslen>0])))
        tslenmin = min(tslenmin, np.min(tslen[tslen>hid]))
    nc.close()
    print(tslenmin)
Lons = np.arange(361)
Lats = np.arange(maxminlat, minmaxlat+1);
vLons, vLats = np.meshgrid(Lons, Lats);  vLons = vLons.flatten(); vLats = vLats.flatten();

nc_target = Dataset(dirReadhigh + 'sp%d_dd10/timeseries_per_location_ddeg%d_sp%d_dd%d_tempres5_ds2.nc'%(sp,ddeg,sp,dd))
idxt = np.where(np.logical_and(nc_target['vLats'][:]>=Lats[0],nc_target['vLats'][:]<=Lats[-1]))
lat = nc_target['lat'][:]; lon = nc_target['lon'][:]
vLons_target =  nc_target['vLons'][:]; vLats_target =  nc_target['vLats'][:];

lsq = np.array([])

#avgd, surf = cm.calc_fields(Lats, Lons, vLats, vLons, name = dirReadhigh + 'timeseries_per_location_ddeg%d_sp%d_dd%d_tempres5.nc'%(ddeg,sp,dd))

#print('average travelled distance in the high res simulation: %f'%np.nanmean(avgd))
#print('average surface area in the high res simulation: %f'%np.nanmean(surf))
#assert False
j=0
for i in allresults:
    if('monmean'==i):
        distance = np.full(len(vLons), np.nan)
        print('monmean')
        ds = Dataset(dirWrite + 'OTs_monmean_sp%d.nc'%(sp), 'w')
        lats = ds.createDimension('lat',len(Lats))
        latitudes = ds.createVariable('Lats', np.float32, ('lat',))
        latitudes[:] = Lats
        lons = ds.createDimension('lon',len(Lons))
        longitudes = ds.createVariable('Lons', np.float32, ('lon',))
        longitudes[:] = Lons
        #cs = ds.createDimension('cs',len(CS))
        OTS = ds.createVariable('Wasserstein distance', np.float32, ('lat','lon',))

        nc_source = Dataset(dirReadhigh + 'sp%d_dd10/timeseries_per_location_ddeg1_sp%d_dd10_tempresmonmean.nc'%(sp,sp))
        lat_s = nc_source['lat'][:]
        lon_s = nc_source['lon'][:]
        vLats_s = nc_source['vLats'][:]
        vLons_s = nc_source['vLons'][:]

        for j in range(Lons.shape[0]-1):
            if(j%30==0): print(j/float(len(Lons)));
            for k in range(Lats.shape[0]):
                idx0 = np.where(np.logical_and(vLons%360==Lons[j]%360,vLats==Lats[k]))
                idxt = np.where(np.logical_and(vLons_target%360==Lons[j]%360,vLats_target==Lats[k]))
                idxs = np.where(np.logical_and(vLons_s%360==Lons[j]%360,vLats_s==Lats[k]))
                latu = lat[idxt[0],:].flatten(); latu = latu[np.where(latu>-500)];
                lasu = lat_s[idxs[0],:].flatten(); lasu = lasu[np.where(lasu>-500)];
                lotu = lon[idxt[0],:].flatten(); lotu = lotu[np.where(lotu>-500)];
                losu = lon_s[idxs[0],:].flatten(); losu = losu[np.where(losu>-500)];
                if(len(lasu)>0 and len(latu>0) and len(losu)>0 and len(lotu)>0):
                    if(not (lasu.mask.all() or latu.mask.all())):
                        lasu = lasu[~lasu.mask][:int(tslenmin)];losu = losu[~losu.mask][:int(tslenmin)];
                        latu = latu[~latu.mask][:int(tslenmin)];lotu = lotu[~lotu.mask][:int(tslenmin)];

                        yt, xt, ys, xs = cm.transform_latlon_to_km_frame(latu, lotu, lasu, losu)

                        t = np.concatenate((xt[:,np.newaxis], yt[:,np.newaxis]),axis=1)
                        s = np.concatenate((xs[:,np.newaxis], ys[:,np.newaxis]),axis=1)

                        distance[idx0] = cm.Wdist(s,t)
    #                print(time.time()-ti)
        OTS[:] = distance.reshape(len(Lats), len(Lons))

        ds.close()
    elif('null'==i):
        distance = np.full(len(vLons), np.nan)
        print('nul test')
        ds = Dataset(dirWrite + 'OTs_nultest.nc', 'w')
        lats = ds.createDimension('lat',len(Lats))
        latitudes = ds.createVariable('Lats', np.float32, ('lat',))
        latitudes[:] = Lats
        lons = ds.createDimension('lon',len(Lons))
        longitudes = ds.createVariable('Lons', np.float32, ('lon',))
        longitudes[:] = Lons
        #cs = ds.createDimension('cs',len(CS))
        OTS = ds.createVariable('Wasserstein distance', np.float32, ('lat','lon',))

        nc_source  = Dataset(dirReadhigh + 'sp%d_dd10/timeseries_per_location_ddeg%d_sp%d_dd%d_tempres5.nc'%(sp,ddeg,sp,dd))
        lat_s = nc_source['lat'][:]
        lon_s = nc_source['lon'][:]
        vLats_s = nc_source['vLats'][:]
        vLons_s = nc_source['vLons'][:]

#        print(np.unique(vLats_s), np.unique(vLats_target))
#        print(np.unique(vLons_s), np.unique(vLons_target))

        for j in range(Lons.shape[0]-1):
            if(j%30==0): print(j/float(len(Lons)));
            for k in range(Lats.shape[0]):
                idx0 = np.where(np.logical_and(vLons%360==Lons[j]%360,vLats==Lats[k]))
                idxt = np.where(np.logical_and(vLons_target%360==Lons[j]%360,vLats_target==Lats[k]))
                idxs = np.where(np.logical_and(vLons_s%360==Lons[j]%360,vLats_s==Lats[k]))
                latu = lat[idxt[0],:].flatten(); latu = latu[np.where(latu>-500)];
                lasu = lat_s[idxs[0],:].flatten(); lasu = lasu[np.where(lasu>-500)];
                lotu = lon[idxt[0],:].flatten(); lotu = lotu[np.where(lotu>-500)];
                losu = lon_s[idxs[0],:].flatten(); losu = losu[np.where(losu>-500)];
                if(len(lasu)>0 and len(latu>0) and len(losu)>0 and len(lotu)>0):
                    if(not (lasu.mask.all() or latu.mask.all())):
                        lasu = lasu[~lasu.mask][:int(tslenmin)];losu = losu[~losu.mask][:int(tslenmin)];
                        latu = latu[~latu.mask][:int(tslenmin)];lotu = lotu[~lotu.mask][:int(tslenmin)];

                        yt, xt, ys, xs = cm.transform_latlon_to_km_frame(latu, lotu, lasu, losu)

                        t = np.concatenate((xt[:,np.newaxis], yt[:,np.newaxis]),axis=1)
                        s = np.concatenate((xs[:,np.newaxis], ys[:,np.newaxis]),axis=1)

                        distance[idx0] = cm.Wdist(s,t)
#                        print(idx0, distance[idx0])
#                        assert False

        OTS[:] = distance.reshape(len(Lats), len(Lons))

#        avgdt, surft = cm.calc_fields(Lats, Lons, vLats, vLons, name = dirReadhigh + 'timeseries_per_location_ddeg%d_sp%d_dd%d_tempres5_ds.nc'%(ddeg,sp,dd), ml=tslenmin)

        ds.close()
        plt.imshow(distance.reshape(len(Lats), len(Lons)))
        plt.show()
    else:
        print('CS: %s'%i)
        distance = np.full(len(vLons), np.nan)
        ds = Dataset(dirWrite + 'OTs_sp%d_cs%s.nc'%(sp,i), 'w')
        lats = ds.createDimension('lat',len(Lats))
        latitudes = ds.createVariable('Lats', np.float32, ('lat',))
        latitudes[:] = Lats
        lons = ds.createDimension('lon',len(Lons))
        longitudes = ds.createVariable('Lons', np.float32, ('lon',))
        longitudes[:] = Lons
        OTS = ds.createVariable('Wasserstein distance', np.float32, ('lat','lon',))

        if(i[-2:]=='gm'):
            cs = np.float(i[:-2])
        else:
            cs = np.float(i)

        if(i[-2:]=='gm'):
            if(cs!=0.25):
                nc_source = nc_target = Dataset(dirReadlow + 'sp%d_dd10/timeseries_per_location_smagorinksi_wn_gm_Cs%.1f_ddeg%d_sp%d_dd%d.nc'%(sp,cs,ddeg,sp,dd))
            else:
                nc_source = nc_target = Dataset(dirReadlow + 'sp%d_dd10/timeseries_per_location_smagorinksi_wn_gm_Cs%.2f_ddeg%d_sp%d_dd%d.nc'%(sp,cs,ddeg,sp,dd))
        else:
            if(cs!=0.25):
                nc_source = nc_target = Dataset(dirReadlow + 'sp%d_dd10/timeseries_per_location_smagorinksi_wn_Cs%.1f_ddeg%d_sp%d_dd%d.nc'%(sp,cs,ddeg,sp,dd))
            else:
                nc_source = nc_target = Dataset(dirReadlow + 'sp%d_dd10/timeseries_per_location_smagorinksi_wn_Cs%.2f_ddeg%d_sp%d_dd%d.nc'%(sp,cs,ddeg,sp,dd))

        lat_s = nc_source['lat'][:]
        lon_s = nc_source['lon'][:]
        vLats_s = nc_source['vLats'][:]
        vLons_s = nc_source['vLons'][:]

        for j in range(Lons.shape[0]):
            if(j%30==0): print(j/float(len(Lons)));
            for k in range(Lats.shape[0]):
    #            ti = time.time()
                idx0 = np.where(np.logical_and(vLons%360==Lons[j]%360,vLats==Lats[k]))
                idxt = np.where(np.logical_and(vLons_target%360==Lons[j]%360,vLats_target==Lats[k]))
                idxs = np.where(np.logical_and(vLons_s%360==Lons[j]%360,vLats_s==Lats[k]))
                latu = lat[idxt[0],:].flatten(); latu = latu[np.where(latu>-500)];
                lasu = lat_s[idxs[0],:].flatten(); lasu = lasu[np.where(lasu>-500)];
                lotu = lon[idxt[0],:].flatten(); lotu = lotu[np.where(lotu>-500)];
                losu = lon_s[idxs[0],:].flatten(); losu = losu[np.where(losu>-500)];
                if(len(lasu)>0 and len(latu>0) and len(losu)>0 and len(lotu)>0):
                    if(not (lasu.mask.all() or latu.mask.all())):
#                        if((latu>0).any()):
#                            print(lotu, losu)
#                            ds.close()
#                            assert False
                        lasu = lasu[~lasu.mask][:int(tslenmin)];losu = losu[~losu.mask][:int(tslenmin)];
                        latu = latu[~latu.mask][:int(tslenmin)];lotu = lotu[~lotu.mask][:int(tslenmin)];

                        yt, xt, ys, xs = cm.transform_latlon_to_km_frame(latu, lotu, lasu, losu)

                        t = np.concatenate((xt[:,np.newaxis], yt[:,np.newaxis]),axis=1)
                        s = np.concatenate((xs[:,np.newaxis], ys[:,np.newaxis]),axis=1)

                        distance[idx0] = cm.Wdist(s,t)
        OTS[:] = distance.reshape(len(Lats), len(Lons))
        lsq = np.append(lsq, np.nanmean(distance))

#        AVGD[:] = np.concatenate((avgdt, np.full((avgdt.shape[0],1),np.nan)),axis=1); SURF[:] = np.concatenate((surft, np.full((surft.shape[0],1),np.nan)),axis=1);
        ds.close()
        j+=1
    #%%


text:
import numpy as np

from test.runtime.frontend_test.tensorflow_test.util import TensorFlowConverter, tf
from test.util import generate_kernel_test_case, wrap_template

@wrap_template
def template(x_shape=[2, 3, 4, 5], pred=True, description: str = ""):
    from tensorflow.python.ops import control_flow_ops

    x = tf.placeholder(np.float32, x_shape)
    y_f, y_t = control_flow_ops.switch(x, pred)
    y = y_t if pred else y_f

    vx = np.random.rand(*x_shape).astype(np.float32)
    with tf.Session() as sess:
        vy, = sess.run([y], {x: vx})
        graph = TensorFlowConverter(sess, batch_size=2).convert([x], [y])

    generate_kernel_test_case(
        description=f"[TensorFlow] Switch {description}",
        graph=graph,
        inputs={graph.inputs[0]: vx},
        expected={graph.outputs[0]: vy},
    )

def test_true():
    template(pred=True)

def test_false():
    template(pred=False)

max_stars_repo_name: steerapi/webdnn

max_issues_repo_name: steerapi/webdnn

max_forks_repo_name: steerapi/webdnn


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Sep 21 10:24:46 2020

@author: jamesjirsa
"""
#need to add database feed

import pandas as pd
import requests
import json
import datetime
import numpy as np
import time

def points_bet(sleep = 1):
    url = ('https://api.il.pointsbet.com/api/v2/competitions/2/events/featured?includeLive=true')
    response = requests.get(url)
    json_data = json.loads(response.text)
    games = json_data['events']
    df = pd.DataFrame()
    for i in range(0,len(games)):
        xtra = games[i]
        del xtra['numberOfSpreadMarkets']
        del xtra['fixedOddsMarkets']
        del xtra['markets']
        del xtra['score']
        del xtra['tags']
        del xtra['eventInfo']
        del xtra['vision']
        df1 = pd.DataFrame.from_dict(xtra)
        df1 = df1.drop('specialFixedOddsMarkets', axis = 1)
        df1 = df1[['timestamp','name', 'key', 'homeTeam', 'awayTeam', 'startsAt']]
        df1 = df1.drop_duplicates()
        for j in range(0, len(games[i]['specialFixedOddsMarkets'])):
            game2 = games[i]['specialFixedOddsMarkets'][j]
            del game2['tags']
            del game2['gameLineMarketName']
            df2 = pd.DataFrame.from_dict(game2)
            df2 = df2.drop_duplicates(subset = ['key'])
            df2 = df2[['key', 'eventClass']]
            df1['temp1'] = i
            df1['temp2'] = j
            df2['temp1'] = i
            df2['temp2'] = j
            fin = pd.merge(df1,df2, on = ['temp1', 'temp2'])
            for k in range(0, len(games[i]['specialFixedOddsMarkets'][j]['outcomes'])):
                game3 = games[i]['specialFixedOddsMarkets'][j]['outcomes'][k]
                df3 = pd.DataFrame.from_dict([game3])
                df3 = df3[['name', 'points', 'price', 'fixedMarketId']]
                df3 = df3.drop_duplicates()
                if len(df.columns) < 0:
                    df3['temp1'] = i
                    df3['temp2'] = j
                    abc = pd.merge(fin, df3, on = ['temp1', 'temp2'])
                    abc = abc.drop(['temp1', 'temp2'], axis = 1)
                    abc = abc.rename(columns = {'key_x' : 'game_id', 'name_x': 'name',
                              'homeTeam': 'homeTeamName', 'awayTeam': 'awayTeamName',
                              'key_y': 'bet_id', 'fixedMarketId': 'id',
                              'eventClass': 'type', 'points': 'line',
                              'price': 'oddsDecimal', 'name_y': 'label'})
                    df = abc

                else:
                    df3['temp1'] = i
                    df3['temp2'] = j
                    abc = pd.merge(fin, df3, on = ['temp1', 'temp2'])
                    abc = abc.drop(['temp1', 'temp2'], axis = 1)
                    abc = abc.rename(columns = {'key_x' : 'game_id', 'name_x': 'name',
                                              'homeTeam': 'homeTeamName', 'awayTeam': 'awayTeamName',
                                              'key_y': 'bet_id', 'fixedMarketId': 'id',
                                              'eventClass': 'type', 'points': 'line',
                                              'price': 'oddsDecimal', 'name_y': 'label', 'startsAt': 'startDate'})
                    df = pd.concat([df,abc])
    df['Book'] = 'PointsBet'
    df['DateTime'] = datetime.datetime.now()
    df = df[['game_id' ,'name', 'homeTeamName', 'awayTeamName','startDate', 'bet_id', 'type',
             'id', 'label', 'line', 'oddsDecimal', 'Book', 'DateTime']]
    ipad = SQL.ipad()
    #sb = SQL.r_sql(ip = ipad, database = 'SportsBet', table = 'NFL_Lines')
    #df = df.compare(sb)
    # add datafeed here
    print('PointsBet Finished')
    time.sleep(sleep)


text:
import torch.nn as nn
import numpy as np
import torch.nn.functional as F

from model_utils import *

class GeneratorConvNet(nn.Module):
    def __init__(self,):
        super(GeneratorConvNet, self).__init__()

        self.ln1 = nn.Linear(128, 6*6*512, bias=False)
        self.reshape = View((512, 6, 6))
        self.bn = nn.BatchNorm2d(512)
        self.relu = nn.ReLU(True)

        self.conv1 = nn.ConvTranspose2d(512, 256, 4, 2, 1, bias=False)
        self.bn1 = nn.BatchNorm2d(256)
        self.relu1 = nn.ReLU(True)

        self.conv2 = nn.ConvTranspose2d(256, 128, 4, 2, 1, bias=False)
        self.bn2 = nn.BatchNorm2d(128)
        self.relu2 = nn.ReLU(True)

        self.conv3 = nn.ConvTranspose2d(128, 64, 4, 2, 1, bias=False)
        self.bn3 = nn.BatchNorm2d(64)
        self.relu3 = nn.ReLU(True)

        self.conv4 = nn.Conv2d(64, 3, 3, 1, 1, bias=False)
        self.tanh = nn.Tanh()

    def forward(self, input):
        output = self.ln1(input)
        output = self.reshape(output)
        output = self.bn(output)
        output = self.relu(output)

        output = self.conv1(output)
        output = self.bn1(output)
        output = self.relu1(output)

        output = self.conv2(output)
        output = self.bn2(output)
        output = self.relu2(output)

        output = self.conv3(output)
        output = self.bn3(output)
        output = self.relu3(output)

        output = self.conv4(output)
        output = self.tanh(output)

        return output

class DiscriminatorConvNet(nn.Module):
    def __init__(self,):
        super(DiscriminatorConvNet, self).__init__()

        self.conv1_1 = nn.Conv2d(3, 64, 3, 1, 1, bias=False)
        self.ln1_1 =  nn.LayerNorm([64,48,48]) #nn.BatchNorm2d(64)
        self.lrelu1_1 = nn.LeakyReLU(0.2, inplace=True)

        self.conv1_2 = nn.Conv2d(64, 64, 4, 2, 1, bias=False)
        self.ln1_2 =  nn.LayerNorm([64,24,24]) #nn.BatchNorm2d(64)
        self.lrelu1_2 = nn.LeakyReLU(0.2, inplace=True)

        self.conv2_1 = nn.Conv2d(64, 128, 3, 1, 1, bias=False)
        self.ln2_1 = nn.LayerNorm([128,24,24]) #nn.BatchNorm2d(128) #
        self.lrelu2_1 = nn.LeakyReLU(0.2, inplace=True)

        self.conv2_2 = nn.Conv2d(128, 128, 4, 2, 1, bias=False)
        self.ln2_2 = nn.LayerNorm([128,12,12]) #nn.BatchNorm2d(128) #
        self.lrelu2_2 = nn.LeakyReLU(0.2, inplace=True)

        self.conv3_1 = nn.Conv2d(128, 256, 3, 1, 1, bias=False)
        self.ln3_1 = nn.LayerNorm([256,12,12]) #nn.BatchNorm2d(256) #
        self.lrelu3_1 = nn.LeakyReLU(0.2, inplace=True)

        self.conv3_2 = nn.Conv2d(256, 256, 4, 2, 1, bias=False)
        self.ln3_2 = nn.LayerNorm([256,6,6]) #nn.BatchNorm2d(256) #
        self.lrelu3_2 = nn.LeakyReLU(0.2, inplace=True)

        ## OLD CODE ##
        self.conv4 = nn.Conv2d(256, 512, 3, 1, 1, bias=False)
        self.gb = GeometricBlock(dim=512, pool=True)
        ############

        #self.conv4 = nn.Conv2d(256, 512, 3, 1, 1, bias=False)
        #self.ln_final = nn.Linear(512*6*6, 1)
        #self.sigm = nn.Sigmoid()

    def forward(self, input):

        output = self.conv1_1(input)
        output = self.ln1_1(output)
        output = self.lrelu1_1(output)

        output = self.conv1_2(output)
        output = self.ln1_2(output)
        output = self.lrelu1_2(output)

        output = self.conv2_1(output)
        output = self.ln2_1(output)
        output = self.lrelu2_1(output)

        output = self.conv2_2(output)
        output = self.ln2_2(output)
        output = self.lrelu2_2(output)

        output = self.conv3_1(output)
        output = self.ln3_1(output)
        output = self.lrelu3_1(output)

        output = self.conv3_2(output)
        output = self.ln3_2(output)
        output = self.lrelu3_2(output)

        ### OLD CODE ####
        output = self.conv4(output)
        output = self.gb(output)
        ##################

        #output = F.adaptive_avg_pool2d(output, (1, 1))
        #output = self.conv4(output)
        #output = output.view(output.size()[0], -1)
        #output = self.ln_final(output)
        #output = self.sigm(output)

        return output


text:
import os
import sys

import tensorflow as tf
import scipy
import numpy as np

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.append(os.path.join(BASE_DIR, '../utils'))

import tf_util
from custom_layers import Scale
from keras.layers import Input, Dense, Convolution2D, MaxPooling2D, AveragePooling2D, ZeroPadding2D, Dropout, Flatten, add, Reshape, Activation
from keras.layers.normalization import BatchNormalization
from keras.models import Model

def placeholder_inputs(batch_size, img_rows=224, img_cols=224, separately=False):
    imgs_pl = tf.placeholder(tf.float32, shape=(batch_size, img_rows, img_cols, 3))
    if separately:
        speeds_pl = tf.placeholder(tf.float32, shape=(batch_size))
        angles_pl = tf.placeholder(tf.float32, shape=(batch_size))
        labels_pl = [speeds_pl, angles_pl]
    labels_pl = tf.placeholder(tf.float32, shape=(batch_size, 2))
    return imgs_pl, labels_pl

def get_resnet(img_rows=224, img_cols=224, separately=False):
    """
    Resnet 152 Model for Keras

    Model Schema and layer naming follow that of the original Caffe implementation
    https://github.com/KaimingHe/deep-residual-networks

    ImageNet Pretrained Weights
    Theano: https://drive.google.com/file/d/0Byy2AcGyEVxfZHhUT3lWVWxRN28/view?usp=sharing
    TensorFlow: https://drive.google.com/file/d/0Byy2AcGyEVxfeXExMzNNOHpEODg/view?usp=sharing

    Parameters:
      img_rows, img_cols - resolution of inputs
      channel - 1 for grayscale, 3 for color
    """
    eps = 1.1e-5
    # Handle Dimension Ordering for different backends
    img_input = Input(shape=(img_rows, img_cols, 3), name='data')

    x = ZeroPadding2D((3, 3), name='conv1_zeropadding')(img_input)
    x = Convolution2D(64, (7, 7), strides=(2, 2), name='conv1', use_bias=False)(x)
    x = BatchNormalization(epsilon=eps, axis=3, name='bn_conv1')(x)
    x = Scale(axis=3, name='scale_conv1')(x)
    x = Activation('relu', name='conv1_relu')(x)
    x = MaxPooling2D((3, 3), strides=(2, 2), name='pool1')(x)

    x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1))
    x = identity_block(x, 3, [64, 64, 256], stage=2, block='b')
    x = identity_block(x, 3, [64, 64, 256], stage=2, block='c')

    x = conv_block(x, 3, [128, 128, 512], stage=3, block='a')
    for i in range(1,8):
      x = identity_block(x, 3, [128, 128, 512], stage=3, block='b'+str(i))

    x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a')
    for i in range(1,36):
      x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b'+str(i))

    x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a')
    x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b')
    x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c')

    x_fc = AveragePooling2D((7, 7), name='avg_pool')(x)
    x_fc = Flatten()(x_fc)
    x_fc = Dense(1000, activation='softmax', name='fc1000')(x_fc)

    model = Model(img_input, x_fc)

    weights_path = 'utils/weights/resnet152_weights_tf.h5'
    assert (os.path.exists(weights_path))
    model.load_weights(weights_path, by_name=True)

    # Truncate and replace softmax layer for transfer learning
    # Cannot use model.layers.pop() since model is not of Sequential() type
    # The method below works since pre-trained weights are stored in layers but not in the model
    x_newfc = AveragePooling2D((7, 7), name='avg_pool')(x)
    x_newfc = Flatten()(x_newfc)
    x_newfc = Dense(256, name='fc8')(x_newfc)
    model = Model(img_input, x_newfc)

    return model

def get_model(net, is_training, add_lstm=False, bn_decay=None, separately=False):
    """ ResNet152 regression model, input is BxWxHx3, output Bx2"""
    net = get_resnet(224, 224)(net)

    if not add_lstm:
        net = tf_util.fully_connected(net, 2, activation_fn=None, scope='fc_final')

    else:
        net = tf_util.fully_connected(net, 784, bn=True,
                                      is_training=is_training,
                                      scope='fc_lstm',
                                      bn_decay=bn_decay)
        net = tf_util.dropout(net, keep_prob=0.7,
                              is_training=is_training,
                              scope="dp1")
        net = cnn_lstm_block(net)

    return net

def cnn_lstm_block(input_tensor):
    lstm_in = tf.reshape(input_tensor, [-1, 28, 28])
    lstm_out = tf_util.stacked_lstm(lstm_in,
                                    num_outputs=10,
                                    time_steps=28,
                                    scope="cnn_lstm")

    W_final = tf.Variable(tf.truncated_normal([10, 2], stddev=0.1))
    b_final = tf.Variable(tf.truncated_normal([2], stddev=0.1))
    return tf.multiply(tf.atan(tf.matmul(lstm_out, W_final) + b_final), 2)

def identity_block(input_tensor, kernel_size, filters, stage, block):
    '''The identity_block is the block that has no conv layer at shortcut
    # Arguments
        input_tensor: input tensor
        kernel_size: defualt 3, the kernel size of middle conv layer at main path
        filters: list of integers, the nb_filters of 3 conv layer at main path
        stage: integer, current stage label, used for generating layer names
        block: 'a','b'..., current block label, used for generating layer names
    '''
    eps = 1.1e-5
    nb_filter1, nb_filter2, nb_filter3 = filters
    conv_name_base = 'res' + str(stage) + block + '_branch'
    bn_name_base = 'bn' + str(stage) + block + '_branch'
    scale_name_base = 'scale' + str(stage) + block + '_branch'

    x = Convolution2D(nb_filter1, (1, 1), name=conv_name_base + '2a', use_bias=False)(input_tensor)
    x = BatchNormalization(epsilon=eps, axis=3, name=bn_name_base + '2a')(x)
    x = Scale(axis=3, name=scale_name_base + '2a')(x)
    x = Activation('relu', name=conv_name_base + '2a_relu')(x)

    x = ZeroPadding2D((1, 1), name=conv_name_base + '2b_zeropadding')(x)
    x = Convolution2D(nb_filter2, (kernel_size, kernel_size),
                      name=conv_name_base + '2b', use_bias=False)(x)
    x = BatchNormalization(epsilon=eps, axis=3, name=bn_name_base + '2b')(x)
    x = Scale(axis=3, name=scale_name_base + '2b')(x)
    x = Activation('relu', name=conv_name_base + '2b_relu')(x)

    x = Convolution2D(nb_filter3, (1, 1), name=conv_name_base + '2c', use_bias=False)(x)
    x = BatchNormalization(epsilon=eps, axis=3, name=bn_name_base + '2c')(x)
    x = Scale(axis=3, name=scale_name_base + '2c')(x)

    x = add([x, input_tensor], name='res' + str(stage) + block)
    x = Activation('relu', name='res' + str(stage) + block + '_relu')(x)
    return x

def conv_block(input_tensor, kernel_size, filters, stage, block, strides=(2, 2)):
    '''conv_block is the block that has a conv layer at shortcut
    # Arguments
        input_tensor: input tensor
        kernel_size: defualt 3, the kernel size of middle conv layer at main path
        filters: list of integers, the nb_filters of 3 conv layer at main path
        stage: integer, current stage label, used for generating layer names
        block: 'a','b'..., current block label, used for generating layer names
    Note that from stage 3, the first conv layer at main path is with subsample=(2,2)
    And the shortcut should have subsample=(2,2) as well
    '''
    eps = 1.1e-5
    nb_filter1, nb_filter2, nb_filter3 = filters
    conv_name_base = 'res' + str(stage) + block + '_branch'
    bn_name_base = 'bn' + str(stage) + block + '_branch'
    scale_name_base = 'scale' + str(stage) + block + '_branch'

    x = Convolution2D(nb_filter1, (1, 1), strides=strides,
                      name=conv_name_base + '2a', use_bias=False)(input_tensor)
    x = BatchNormalization(epsilon=eps, axis=3, name=bn_name_base + '2a')(x)
    x = Scale(axis=3, name=scale_name_base + '2a')(x)
    x = Activation('relu', name=conv_name_base + '2a_relu')(x)

    x = ZeroPadding2D((1, 1), name=conv_name_base + '2b_zeropadding')(x)
    x = Convolution2D(nb_filter2, (kernel_size, kernel_size),
                      name=conv_name_base + '2b', use_bias=False)(x)
    x = BatchNormalization(epsilon=eps, axis=3, name=bn_name_base + '2b')(x)
    x = Scale(axis=3, name=scale_name_base + '2b')(x)
    x = Activation('relu', name=conv_name_base + '2b_relu')(x)

    x = Convolution2D(nb_filter3, (1, 1), name=conv_name_base + '2c', use_bias=False)(x)
    x = BatchNormalization(epsilon=eps, axis=3, name=bn_name_base + '2c')(x)
    x = Scale(axis=3, name=scale_name_base + '2c')(x)

    shortcut = Convolution2D(nb_filter3, (1, 1), strides=strides,
                             name=conv_name_base + '1', use_bias=False)(input_tensor)
    shortcut = BatchNormalization(epsilon=eps, axis=3, name=bn_name_base + '1')(shortcut)
    shortcut = Scale(axis=3, name=scale_name_base + '1')(shortcut)

    x = add([x, shortcut], name='res' + str(stage) + block)
    x = Activation('relu', name='res' + str(stage) + block + '_relu')(x)
    return x

def get_loss(pred, label, l2_weight=0.0001):
    diff = tf.square(tf.subtract(pred, label))
    train_vars = tf.trainable_variables()
    l2_loss = tf.add_n([tf.nn.l2_loss(v) for v in train_vars[1:]]) * l2_weight
    loss = tf.reduce_mean(diff + l2_loss)
    tf.summary.scalar('l2 loss', l2_loss * l2_weight)
    tf.summary.scalar('loss', loss)

    return loss

def summary_scalar(pred, label):
    threholds = [5, 4, 3, 2, 1, 0.5]
    angles = [float(t) / 180 * scipy.pi for t in threholds]
    speeds = [float(t) / 20 for t in threholds]

    for i in range(len(threholds)):
        scalar_angle = "angle(" + str(angles[i]) + ")"
        scalar_speed = "speed(" + str(speeds[i]) + ")"
        ac_angle = tf.abs(tf.subtract(pred[:, 1], label[:, 1])) < threholds[i]
        ac_speed = tf.abs(tf.subtract(pred[:, 0], label[:, 0])) < threholds[i]
        ac_angle = tf.reduce_mean(tf.cast(ac_angle, tf.float32))
        ac_speed = tf.reduce_mean(tf.cast(ac_speed, tf.float32))

        tf.summary.scalar(scalar_angle, ac_angle)
        tf.summary.scalar(scalar_speed, ac_speed)

def resize(imgs):
    batch_size = imgs.shape[0]
    imgs_new = []
    for j in range(batch_size):
        img = imgs[j,:,:,:]
        new = scipy.misc.imresize(img, (224, 224))
        imgs_new.append(new)
    imgs_new = np.stack(imgs_new, axis=0)
    return imgs_new

if __name__ == '__main__':
    with tf.Graph().as_default():
        inputs = tf.zeros((32, 224, 224, 3))
        outputs = get_model(inputs, tf.constant(True))
        print(outputs)

max_stars_repo_name: cclauss/DBNet

max_issues_repo_name: cclauss/DBNet

max_forks_repo_name: cclauss/DBNet

reason: import numpy,import scipy


text:
#!/usr/bin/python3

import random as rdm
import numpy as np

class Block:

    def __init__(self, id, neighbors = None):
        self.id = id
        self.neighbors = [] if neighbors is None else neighbors

    def look_around(self):
        return [tuple([x+(i==ind)*incr for i,x in enumerate(self.id)])
                for ind in range(len(self.id)) for incr in (-1,1)]

    def make_connection(self, block):
        if block not in self.neighbors:
            self.neighbors.append(block)
        block.be_connected(self)

    def be_connected(self, block):
        if block not in self.neighbors:
            self.neighbors.append(block)

class Wall(Block):

    def __init__(self, id, neighbors = None):
        super().__init__(id, neighbors)

    def __str__(self):
        return '0'

    def __repr__(self) -> str:
        return str({'id':self.id})

class Way(Block):

    def __init__(self, id, value, neighbors = None):
        super().__init__(id, neighbors)
        for n in self.neighbors:
            self.make_connection(n)
        self.value = value

    def break_wall(self, wall:Wall):
        if isinstance(wall, Wall):
            if wall in self.neighbors:
                return Way(wall.id, self.value, wall.neighbors)
            else:
                raise ValueError('{} is not a neighbor'.format(wall))
        else:
            raise TypeError('{}'.format(type(wall)))

    def __str__(self):
        # return str(self.value)
        return ' '

    def __repr__(self) -> str:
        return str({'id':self.id, 'value':self.value})

    @property
    def value(self):
        return self.__value

    @value.setter
    def value(self, val):
        self.__value = val
        for n in self.neighbors:
            if isinstance(n, Way) and n.value != val:
                n.value = val

class Labyrinth:

    def __init__(self, shape, nb_exit, nb_internal_break):
        self.__shape = tuple(shape)
        self.__labyrinth = {}

        indexes = self.generate_indexes()

        cpt = 1
        prod = lambda li : li[0] if len(li) == 1 else li[0]*prod(li[1:])
        for ind in indexes:
            if prod(ind)%2 == 0:
                block = Wall(ind)
            else:
                block = Way(ind, cpt)
                cpt += 1

            self[ind] = block

            for n in block.look_around():
                if self.check_index(n) and n in self:
                    block.make_connection(self[n])

        self.__aff = len(str(cpt))

        exit_walls = self.find_exit_walls()

        e = 0
        while exit_walls and e < nb_exit:
            exit_wall = exit_walls.pop(rdm.choice(range(len(exit_walls))))
            exit_way = [self[ind] for ind in exit_wall.look_around()
                        if self.check_index(ind)
                        and isinstance(self[ind],Way)][0]

            self[exit_wall.id] = exit_way.break_wall(exit_wall)
            e+=1

        internal_walls = self.find_internal_walls()
        intra_area_walls = []
        while self.get_number_of_areas() != 1:
            wall = internal_walls.pop(rdm.choice(range(len(internal_walls))))
            way_1, way_2 = [block for block in wall.neighbors
                            if isinstance(block,Way)]

            if way_1.value != way_2.value:
                self[wall.id] = way_1.break_wall(wall)
            else:
                intra_area_walls.append(wall)

        internal_walls.extend(intra_area_walls)
        b = 0
        while internal_walls and b < nb_internal_break:
            wall = internal_walls.pop(rdm.choice(range(len(internal_walls))))
            way = rdm.choice([block for block in wall.neighbors
                                if isinstance(block,Way)])
            self[wall.id] = way.break_wall(wall)
            b += 1

    def generate_indexes(self, shape = None):
        if shape is None:
            shape = self.shape

        prod = lambda li : li[0] if len(li) == 1 else li[0]*prod(li[1:])
        rep = [1]+[prod(shape[0:i]) for i in range(1,len(shape))]
        return [tuple([j%(rep[i]*shape[i])//rep[i]
                 for i in range(len(shape))])
                for j in range(prod(shape))]

    def check_index(self, index):
        return False if [x for i,x in enumerate(index)
                         if (x<0 or x>=self.shape[i])] else True

    def find_exit_walls(self):
        prod = lambda li : li[0] if len(li) == 1 else li[0]*prod(li[1:])
        return [self[ind] for ind in self
                if len([x for i,x in enumerate(ind)
                        if x in (0, self.shape[i]-1)]) == 1
                and prod([x for i,x in enumerate(ind)
                          if x not in (0, self.shape[i]-1)])%2 != 0
                and isinstance(self[ind], Wall)]

    def find_internal_walls(self):
        prod = lambda li : li[0] if len(li) == 1 else li[0]*prod(li[1:])
        return [self[ind] for ind in self if prod(ind)%2 == 0
                and not [x for i,x in enumerate(ind)
                         if x in (0,self.shape[i]-1)]
                and [n for n in self[ind].neighbors if isinstance(n,Way)]]

    def get_number_of_areas(self):
        return len(set([self[ind].value for ind in self
                        if isinstance(self[ind],Way)]))

    def to_array(self):
        array = np.zeros(self.shape)
        for ind in self:
            array[ind] = 0 if isinstance(self[ind],Way) else 1

        return array

    def __str__(self):
        if len(self.shape) > 2:
            plans = self.generate_indexes(self.shape[2:])
            p_indexes = [sorted(sorted([ind for ind in self if ind[2:] == p ],
                                       key=lambda x:x[0]),
                                key=lambda x:x[1])
                         for p in plans]
        else:
            p_indexes = [sorted(sorted(list(self), key=lambda x:x[0]),
                                key=lambda x:x[1])]

        txt_li = []
        for p in p_indexes:
            txt_li.append('\n')
            txt_li.append(
                '(., .{})'.format(''.join([', {}'.format(x)
                                           for x in p[0][2:]]))
            )

            txt_li.append(
                '\n'.join(
                    [' '.join(
                        ['{{:>{}}}'.format(1).format(
                            str(self[p[i+j*self.shape[0]]]))
                         for i in range(self.shape[0])])
                     for j in range(self.shape[1])])
            )

        return '\n'.join(txt_li)

    def __getitem__(self, key):
        return self.__labyrinth[key]

    def __setitem__(self, key, block:Block):
        self.__labyrinth[key] = block

    def __contains__(self, index):
        return True if index in self.__labyrinth else False

    def __iter__(self):
        return iter(self.__labyrinth)

    def __next__(self):
        return next(self.__labyrinth)

    @property
    def shape(self):
        return self.__shape


text:
import numpy as np
from scipy.linalg import solve

def gaussSeidel(A, b, x, n):

    L = np.tril(A)
    U = A - L
    for i in range(n):
        x = np.dot(np.linalg.inv(L), b - np.dot(U, x))
        print ('\n','Iter ', i, ':')
        print(x)
    return x

'''___MAIN___'''

A = np.array([[1/2, 1/3, 1/4], [1/3, 1/4, 1/5], [1/4, 1/5, 1/6]])
b = [0.95, 0.67, 0.52]
x = [1, 1, 1]

n = 100

gaussSeidel(A, b, x, n)

exit()
#print (solve(A, b))

reason: import numpy,from scipy


text:
import glob

from six.moves import cPickle as pickle
from sklearn import preprocessing
import numpy as np
import scipy.misc
# "
#
# a = {"sdfg", "sdfsdf"}
#
# with open("images.pickle", 'wb') as f:
#     pickle.dump(a, f, protocol=pickle.HIGHEST_PROTOCOL)
#
# with open("images.pickle", 'rb') as ff:
#     bb = pickle.load(ff)
#from cifar import download_and_convert_satellite

def createFiles(path, classes):
    datalist = {}
    datalist["images"] = []
    datalist["labels"] = []
    for filename in glob.iglob(path, recursive=True):

        image = scipy.misc.imread(filename).astype(np.uint8)
        # image = tf.gfile.FastGFile(filename, 'rb').read()
        currentClass = ""
        for cla in classes:
            if cla in filename:
                currentClass = cla
                break
        if currentClass == "":
            print("NO claass is foune rggreub")
        else:
            datalist["images"].append(image.flatten())
            datalist["labels"].append(classes.index(currentClass))
    return datalist

classesList = ["agricultural",'airplane','baseballdiamond','beach','buildings','chaparral'
    ,'denseresidential','forest','freeway','golfcourse','harbor','intersection','mediumresidential',
              'mobilehomepark','overpass','parkinglot','river','runway','sparseresidential',
              'storagetanks','tenniscourt']

train = False
if train:
    lb = preprocessing.LabelBinarizer()
    datapath = "/data/images/uc_train_256_data/**.jpg"
    datalist = createFiles(datapath, classesList)
    with open("/data/satellitegpu/satellite_train.pickle", 'wb') as f:
         pickle.dump(datalist, f, protocol=pickle.HIGHEST_PROTOCOL)
    #download_and_convert_satellite.run("/data/satellitegpu/")
else:
    lb = preprocessing.LabelBinarizer()
    datapath = "/data/images/uc_test_256/**.jpg"
    datalist = createFiles(datapath, classesList)
    with open("/data/satellitegpu/satellite_test.pickle", 'wb') as f:
        pickle.dump(datalist, f, protocol=pickle.HIGHEST_PROTOCOL)
    #download_and_convert_satellite.run("/data/satellitegpu/", train=False)

max_stars_repo_name: GPrathap/DCGAN-Tensorflow

max_issues_repo_name: GPrathap/DCGAN-Tensorflow

max_forks_repo_name: GPrathap/DCGAN-Tensorflow


text:
import os
import ctypes
import numpy
from numpy.ctypeslib import ndpointer
import pkg_resources
import platform
import struct
import enum
import logging
from collections import deque
import sys

class CustomExitCodes (enum.Enum):

    STATUS_OK = 0
    TARGET_PROCESS_IS_NOT_CREATED_ERROR = 1
    PROCESS_MONITOR_ALREADY_RUNNING_ERROR = 2
    PROCESS_MONITOR_IS_NOT_RUNNING_ERROR = 3
    GENERAL_ERROR = 4
    PATH_NOT_FOUND_ERROR = 5
    TARGET_PROCESS_WAS_TERMINATED_ERROR = 6

class InjectionError (Exception):
    def __init__ (self, message, exit_code):
        detailed_message = '%s:%d %s' % (CustomExitCodes (exit_code).name, exit_code, message)
        super (InjectionError, self).__init__ (detailed_message)
        self.exit_code = exit_code

class InjectorDLL (object):

    __instance = None

    @classmethod
    def get_instance (cls):
        if cls.__instance is None:
            if platform.system () != 'Windows':
                raise Exception ("For now only Windows is supported, detected platform is %s" % platform.system ())
            cls.__instance = cls ()
        return cls.__instance

    def __init__ (self):
        if struct.calcsize ("P") * 8 == 64:
            self.lib = ctypes.cdll.LoadLibrary (pkg_resources.resource_filename (__name__, os.path.join ('lib', 'DLLInjection64.dll')))
        else:
            self.lib = ctypes.cdll.LoadLibrary (pkg_resources.resource_filename (__name__, os.path.join ('lib', 'DLLInjection32.dll')))

        # start monitoring
        self.StartMonitor = self.lib.StartMonitor
        self.StartMonitor.restype = ctypes.c_int
        self.StartMonitor.argtypes = [
            ctypes.c_char_p,
            ctypes.c_char_p
        ]

        # stop monitorring
        self.ReleaseResources = self.lib.ReleaseResources
        self.ReleaseResources.restype = ctypes.c_int
        self.ReleaseResources.argtypes = []

        # set log level
        self.SetLogLevel = self.lib.SetLogLevel
        self.SetLogLevel.restype = ctypes.c_int
        self.SetLogLevel.argtypes = [
            ctypes.c_int
        ]

        # get pid
        self.GetPid = self.lib.GetPid
        self.GetPid.restype = ctypes.c_int
        self.GetPid.argtypes = [
            ndpointer (ctypes.c_int64)
        ]

        # send message
        self.SendMessageToOverlay = self.lib.SendMessageToOverlay
        self.SendMessageToOverlay.restype = ctypes.c_int
        self.SendMessageToOverlay.argtypes = [
            ctypes.c_char_p
        ]

        # run process
        self.RunProcess = self.lib.RunProcess
        self.RunProcess.restype = ctypes.c_int
        self.RunProcess.argtypes = [
            ctypes.c_char_p,
            ctypes.c_char_p,
            ctypes.c_char_p
        ]

def start_monitor (process_name):
    logging.warning ('For Steam Games ensure that there is steam_appid.txt file in \%SteamFolder\%\\steamapps\\common\\\%GameName\% with correct appid! You can get app_id here https://steamdb.info/search/')
    location = os.path.abspath (os.path.dirname (pkg_resources.resource_filename (__name__, os.path.join ('lib', 'GameOverlay64.dll'))))
    res = InjectorDLL.get_instance ().StartMonitor (process_name.encode (), location.encode ())
    if res != CustomExitCodes.STATUS_OK.value:
        raise InjectionError ('start process creation monitoring error please check logs', res)

def release_resources ():
    res = InjectorDLL.get_instance ().ReleaseResources ()
    if res != CustomExitCodes.STATUS_OK.value:
        raise InjectionError ('stop monitoring error', res)

def set_log_level (level):
    res = InjectorDLL.get_instance ().SetLogLevel (level)
    if res != CustomExitCodes.STATUS_OK.value:
        raise InjectionError ('failed to set log level', res)

# log level for core module of this library, not related to python logging
def enable_monitor_logger ():
    set_log_level (2)

def disble_monitor_logger ():
    set_log_level (6)

def enable_dev_logger ():
    set_log_level (0)

def get_pid ():
    pid = numpy.zeros (1).astype (numpy.int64)
    res = InjectorDLL.get_instance ().GetPid (pid)
    if res != CustomExitCodes.STATUS_OK.value:
        raise InjectionError ('Callback has not been called yet', res)
    return pid[0]

def send_message (message):
    if not hasattr (send_message, 'message_queue'):
        send_message.message_queue = deque (maxlen = 5)
    send_message.message_queue.append (message)
    acum_message = ''
    for msg in send_message.message_queue:
        acum_message = acum_message + msg + '\n'
    res = InjectorDLL.get_instance ().SendMessageToOverlay (acum_message.encode ())
    if res != CustomExitCodes.STATUS_OK.value:
        raise InjectionError ('failed to send message', res)

def write_app_id (file_path, app_id):
    logging.info ('writing %s to %s' % (str (app_id), file_path))
    with open (file_path, 'w') as f:
        f.write (str(app_id))

def run_process (exe_path, exe_args = "", steam_app_id = None):
    if steam_app_id is None:
        logging.warning ('For Steam Games please provide app id or ensure that there is steam_appid.txt file in game folder! You can get app_id here https://steamdb.info/search/')
    elif not os.path.isabs (exe_path):
        logging.warning ('to create steam_appid.txt file please provide full path to executable')
        raise Exception ('please provide full path')

    game_dir = os.path.abspath (os.path.dirname (exe_path))
    steam_app_id_file = os.path.join (game_dir, 'steam_appid.txt')
    write_app_id (steam_app_id_file, steam_app_id)

    location = os.path.abspath (os.path.dirname (pkg_resources.resource_filename (__name__, os.path.join ('lib', 'GameOverlay64.dll'))))
    res = InjectorDLL.get_instance ().RunProcess (exe_path.encode (), exe_args.encode(), location.encode ())
    if res != CustomExitCodes.STATUS_OK.value:
        raise InjectionError ('failed to run process please check logs', res)

class OvelrayLogHandler (logging.Handler):

    def __init__ (self):
        logging.Handler.__init__(self)

    def emit (self, record):
        try:
            send_message (self.format (record))
        except InjectionError as inj_err:
            # no need to notify about excpetion here cause it is implemented in cpp code
            pass
        except BaseException as e:
            self.handleError (record)

reason: import numpy,from numpy