text:
#!"C:\Users\Satan\Informatica\Python\sci-env\Scripts\python.exe"
# -*- coding: utf-8 -*-
"""
Created on Sat Nov 17 19:30:17 2018

@author: Stan

This module defines and models the network. It contains a list of nodes and can
graphically represent those.

Requirements:
    - Energy consumption: star network??
    - Cluster head communicates over LoRa to base
    - Show differences in energy consumption.
    - Transmit power of 0 dBm (1 mW)
    - Payload 10 Bytes for sensors and 60 for Cluster Heads
    - Frequencies: 868 MHz for LoRa and 2.4GHz for BLE

"""

# Python imports

import click
import matplotlib.pyplot as plt
from matplotlib.patches import Circle, Arrow
import numpy as np
from functools import partial
from pygame import mixer

import config

# Project imports
from node import (SensorNode, BLE, ON)

mixer.init()
mixer.music.load("mario_1-up_mushroom.mp3")

#=============================================
#   GLOBAL PARAMETERS
#=============================================

width = (config.number_of_sensors-1) * config.LANTERN_SEPARATION
                    # Dimensions of the network. 10 sensors, so 9 times
                    # internal distance.
height = 0
cell_width = config.LANTERN_SEPARATION
cell_height = 0

import traffic

#=============================================
#   NETWORK & TRAFFIC GENERATION FUNCTIONS
#=============================================

def create_star():
    '''
    The standard network with constant separation between nodes.
    '''
    sens = []
    x_locs = np.linspace(-width/2,width/2, config.number_of_sensors)
    y_locs = np.zeros(1)

    for i,x in enumerate(x_locs):
        for y in y_locs:
            sens.append(SensorNode(i,x,y, transmission_mode=BLE, packet_rate=config.packet_rate))
    return sens

def spawn_vehicles():
    """
    Is called every tick. May or may not return a car to spawn and append to
    the traffic list.
    return: a Car or None.
    """
    for i,(car, time) in enumerate(config.spawn_list):
        if config.time > time:
            config.spawn_list.pop(i)
            config.cars.append(car)

def despawn_vehicle(car):
    config.cars.remove(car)

#=============================================
#   ANALYSIS FUNCTIONS
#=============================================

circles = []
car_circs = []
sim_fig,sim_ax = None,None

def sim_vis():
    global sim_fig,sim_ax
    sim_fig,sim_ax = plt.subplots()
    circles.clear()
    car_circs.clear()
    for sensor in config.sensors:
        circle = Circle(xy=sensor.loc, radius=5, color='grey')
        circles.append(circle)
        sim_ax.add_artist(circle)

    for car,_ in config.spawn_list:
        circle2 = Circle(xy=car.loc,radius=2,color='blue')
        car_circs.append(circle2)
        sim_ax.add_artist(circle2)

    sim_ax.set_xlim((-width-cell_width, width+cell_width))
    sim_ax.set_ylim((-width/3, width/3))
    sim_ax.set_aspect('equal')

def update_plot():
    for sensor, circle in zip(config.sensors, circles):
        circle.set_color('red') if sensor.state == ON else circle.set_color('grey')

    for car,circle in zip(config.cars, car_circs):
        circle.center = tuple(car.loc)

    plt.title((f'Time = {config.time:.1f}'))
    #plt.show()
    sim_fig.canvas.draw()
    sim_fig.canvas.flush_events()

def visualize():
    """
    Plots the nodes. Not the clusterheads yet. Fix if needed
    """
    fig, ax = plt.subplots()
    sensors = config.sensors

    plt.axis('equal')

    # Plot sensors
    xs = np.array([sensor.loc[0] for sensor in sensors])
    ys = np.array([sensor.loc[1] for sensor in sensors])
    heads = np.array([True if sensor.is_head else False for sensor in sensors])
    bles = np.array([1 if sensor.transmission_mode==BLE else 0 for sensor in sensors])

    plt.plot(xs[(heads==False) & (bles==True)],
            ys[(heads==False) & (bles==True)], 'b.')

    # Plot cross
#    plt.plot([-width/2,width/2], [0,0], 'y')
#    plt.plot([0,0], [-height/2, height/2], 'y')
    # Plot border
#    plt.plot([-width/2, width/2, width/2,-width/2,-width/2],
#             [-height/2,-height/2,height/2,height/2,-height/2], linewidth=3)

    plt.xlabel('x (m)')
    plt.ylabel('y (m)')
    plt.title(f'Map of Lamppost nodes')

    #plt.legend(('BLE-enabled sensors', 'LoRa7 sensors', 'LoRa 8 sensors', 'Cluster-heads'))
    plt.show()

# =============================================================================
# def calc_range(func, threshold):
#     '''
#     Calculates the highest distance in meters for which lora is below the
#     threshold.
#     '''
#     d = 500
#     # Establish upper bound
#     while func(d) < threshold:
#         d *= 2
#     max_d = d
#     min_d = 0
#     d = (min_d+max_d)/2
#     while max_d-min_d > 1:
#         if func(d) > threshold:
#             max_d = d
#
#         else:
#             min_d = d
#         d = (min_d+max_d)/2
#     return np.floor(min_d)
# =============================================================================

def plot_set(setname):

    # !!! Legend does not match

    data_star_urban = extract_data(setname)

    #Markers: s(quare) for urban ^ for rural green for smart, red for star

    if data_star_urban:
        # normal total je weet zelf.
        normal_total = config.end*config.POWER_ON*config.number_of_sensors
        plt.plot(config.density_factor*2262, 100-(data_star_urban[0]/normal_total*100), 'rs')
        plt.plot(plt.xlim(), [100]*2, 'r--')

    plt.title(f"Energy savings (simulated for 60 seconds)")
    plt.xlabel('Cars per hour')
    plt.ylabel('Energy saved (%)')
    plt.show()

def extract_data(setname):
    prefix = setname + '/sim_'
    total_energy = np.zeros(len(config.density_factor))
    try:
        for i in range(len(config.density_factor)):
            print(f'Loading {prefix+str(i)}.npz...')
            arc = np.load(prefix+str(i)+'.npz')
            cons = arc.f.result
    #        bles,heads =  arc.f.bles, arc.f.heads
            total_energy[i] = np.sum(cons)

        return total_energy,
    except:
        return None

def plot_energy(setname):
    """
    Plots the energy of a given situation compared to a single variant.
    Total energy is extracted by the given setname.
    """

    total_energy, = extract_data(setname)

    plt.plot(config.density_factor, total_energy, 'r.')
    plt.title(f"Energy consumptions various traffic densities")
    plt.xlabel('Variant')
    plt.ylabel('Energy (J)')

def simulate(urban=False, vis=False):
    '''
    Simulates one run of the program and returns arrays of data
    '''
    dt = config.dt
    t = 0
    try:
        while config.time < config.end:
            for sensor in config.sensors:
                sensor.tick(dt)
            for car in config.cars:
                car.tick(dt)
            spawn_vehicles()
            if not t%10 and vis: update_plot()
            config.time += dt
            t += 1
    except KeyboardInterrupt:
        print('Simulation aborted!')
    return np.array([sensor.energy_cons for sensor in config.sensors])

def show_energy_consumption(cons):
    plt.figure()
    sensors = config.sensors
    bles = np.array([1 if sensor.transmission_mode==BLE else 0 for sensor in sensors])
    heads = np.array([sensor.is_head for sensor in sensors])

    plt.plot(np.arange(len(cons))[bles==1],cons[bles==1] , 'b.')
    plt.plot(np.arange(len(cons))[heads==1], cons[heads==1] , 'ro', fillstyle='none')
    plt.plot(np.arange(len(cons))[(bles==0)],cons[(bles==0)] , 'g.')

    plt.plot(plt.xlim(), [config.end*config.POWER_ON]*2, 'r--')

    plt.title('Consumed energy per node')
    plt.xlabel('Node ID')
    plt.ylabel('Total consumed energy in J')
    plt.legend(['Lampposts', 'Always-on-consumption'])

    #head_avg = np.mean(cons[heads])
    node_avg = np.mean(cons[heads==False])
    normal_total = config.end*config.POWER_ON*config.number_of_sensors
    percentage = np.sum(cons)/normal_total * 100
    #print(f'Average energy consumption of cluster heads: {head_avg} J')
    print(f'Average energy consumption of nodes: {node_avg} J')
    print(f'Energy consumption: {np.sum(cons):.2f} / {normal_total} ({percentage:.1f}%)J')

def show_energy_consumption_from_file(fname):
    arc = np.load(fname)
    cons,bles,heads = arc.f.result, arc.f.bles,arc.f.heads
    plt.plot(np.arange(len(cons))[bles==1],cons[bles==1] , 'b.')
    plt.plot(np.arange(len(cons))[heads==1], cons[heads==1] , 'ro', fillstyle='none')
    plt.plot(np.arange(len(cons))[(bles==0) ],cons[(bles==0)] , 'g.')

    plt.plot(plt.xlim(), [config.end*config.POWER_ON]*2, 'r--')

    plt.title('Consumed energy per node')
    plt.xlabel('Node ID')
    plt.ylabel('Total consumed energy in J')
    plt.legend(('BLE nodes'))

    node_avg = np.mean(cons[heads==False])
    normal_total = config.end*config.POWER_ON*config.number_of_sensors
    percentage = np.sum(cons)/normal_total * 100
    print(f'Average energy consumption of nodes: {node_avg} J')
    print(f'Energy consumption: {np.sum(cons):.2f} / {normal_total} ({percentage:.1f}%)J')

#def save_consumption(filename):
#    sensors = config.sensors
#    head_cons = np.array([sensor.energy_cons for sensor in sensors if sensor.is_head])
#    node_cons = np.array([sensor.energy_cons for sensor in sensors if not sensor.is_head])
#    np.savez(filename, head_cons=head_cons, node_cons=node_cons)

@click.command()
@click.option('--vis', is_flag=True, default=False, help='Creates a map of the nodes')
@click.option('--sim', is_flag=True, default=False, help='Run a Simulation')
@click.option('--cons-vis', is_flag=True, default=False, help='Plots the '
                 + 'consumption of the simulation. Must be used with --sim')
@click.option('--set','_set', default=None, help='Selects a set to operate on')
def cli(vis, sim, cons_vis, _set):
    for i in range(len(config.density_factor)):
        config.current_density_factor = config.density_factor[i]
        run_sim(vis, sim,cons_vis, _set, i)

    mixer.music.play()

def run_sim(vis, sim, cons_vis, _set, i):
    config.init()
    config.sensors = create_star()
    traffic.random_amount_oneway(int(2262/3600*config.end*config.current_density_factor))

    if sim:
        if vis:
            sim_vis()

        name = 'sim_'
        result = simulate(vis=vis)
        name += str(i)

        bles = np.array([1 if sensor.transmission_mode==BLE else 0 for sensor in config.sensors])
        heads = np.array([sensor.is_head for sensor in config.sensors])

        np.savez(name, result=result, bles=bles, heads=heads)

    if vis:
        if _set:
            plot_set(_set)
        elif not sim:
            visualize()
    if cons_vis:
        #Visualize consumption. Must be used in conjunction with sim.
        show_energy_consumption(result)

    # result,bles,sf7s,heads = arc.f.result, arc.f.bles, arc.f.sf7s, arc.f.heads

if __name__ == '__main__':
    cli()

max_stars_repo_name: stan-v/smart-lamppost-simulation

max_issues_repo_name: stan-v/smart-lamppost-simulation

max_forks_repo_name: stan-v/smart-lamppost-simulation


text:
from random import randint
from sympy import sqrt, expand
from cartesian import *

def perpendicular(P1, P2, P3, subs):
    return expand(fraction(cancel((P1[0] - P2[0])*P3[0] + (P1[1] - P2[1])*P3[1]))[0].subs(subs)) == 0

def sub_y(x, y):
    return y, (1 - randint(0, 1)*2)*sqrt(1 - x**2)

def main():
    # https://imomath.com/index.cgi?page=polePolarBrianchonBrokard (Theorem 9)
    a, b, c, d, e, f, g, h = symbols('a, b, c, d, e, f, g, h')
    A, B, C, D = (a, b), (c, d), (e, f), (g, h)
    AB, AC, AD, BC, BD, CD = line(A, B), line(A, C), line(A, D), line(B, C), line(B, D), line(C, D)
    E, F, G = intersect(AB, CD), intersect(AD, BC), intersect(AC, BD)
    print('E:', E)
    print('F:', F)
    print('G:', G)
    subs = [sub_y(a, b), sub_y(c, d), sub_y(e, f), sub_y(g, h)]
    print('Is EF perpendicular to OG?', perpendicular(E, F, G, subs))
    print('Is FG perpendicular to OE?', perpendicular(F, G, E, subs))
    print('Is GE perpendicular to OF?', perpendicular(G, E, F, subs))

if __name__ == '__main__':
    main()

ext: py

max_stars_repo_name: auntyellow/math

max_issues_repo_name: auntyellow/math

max_forks_repo_name: auntyellow/math

reason: from sympy


text:
# !/usr/bin/env python
#
# 'localisation.py' has a simple implementation of calculation for
#  disorder-averaged amplitudes of a 1d-tight binding model with
#  the nearest neighbor couplings being perturbed by disorder.
#
#
# MIT License. Copyright (c) 2020 Vijay Mocherla
#
# Source code at
# <htts://github.com/vijaymocherla/Localisation-in-Open-quantum-systems.git>

# Importing a few packages
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import sys
from numpy import linalg
from scipy import integrate
import time as Tclock

def Hamiltonian(N,V):
    """ Generates the 'N'-dimensional Hamiltonian matrix of
        a 1-d tight binding model for an array nearest neighbor site couplings 'V'
    """
    H = np.zeros((N,N))
    H[0][1] = V[0]
    H[N-1][N-2] = V[N-1]
    for i in range(1,N-1):
        H[i][i+1] = V[i]
        H[i][i-1] = V[i]
    H = np.eye(N) + H
    return(H)

def conf_run(N,seed_loc,params):
    """  This function returns the matrix object at every time point for a certain realisation of disorder """
    K,T,tSteps = params
    x = 1
    time = np.linspace(0,T,tSteps)
    V = 2*np.random.random(N) + (x-1)
    vals,vecs = linalg.eigh(Hamiltonian(N,V))
    trans_prob = lambda t : (np.abs( np.array( [ vecs[i][seed_loc-1]*vecs[i]*np.exp(1j*vals[i]*t) for i in range(N)] ) )**2).sum(axis=0)
    data = np.array([trans_prob(t) for t in time])
    return(data)

def disorder_avg(N,seed_loc,params):
    K,T,tSteps = params
    time = np.linspace(0,T,tSteps)
    davg_tps = np.zeros((tSteps,N))
    st1 = Tclock.time()
    # In the following loop, we generation random outcomes K times and add to davg_tps.
    # davg_tps is then averaged by total no. cycles.
    for i in range(K):
        #st2 = Tclock.time()
        davg_tps+=conf_run(N,seed_loc,params)
        #e2 = Tclock.time()
        #print(e2-st2)

    davg_tps = davg_tps/K

    e1 = Tclock.time()
    print('time per cycle : ',(e1-st1)/K )
    print('time entire run in mins : ', (e1-st1)/60 )
    return(davg_tps,time)

def main(N,params):
    K,T,tSteps = params
    data,time = disorder_avg(N,int(N/2 +1),params)
    data,time = disorder_avg(N,int(6),params)
    sort = [data[:,i:i+1].flat for i in range(N)]
    time_avg  = [np.trapz(array,time)/params[1] for array in sort]
    print('Total Probability for consistency check, (this should be 1.0):',sum(time_avg))
    #population_data[(N,100)] = time_avg
    np.savetxt('N_'+str(N)+'_conf_'+str(K)+'_T_'+str(T)+'data',data)
    fig = plt.figure()
    plt.plot(range(1,N+1),time_avg,':',marker='o')
    plt.xlabel('n')
    plt.ylabel(r'$\langle P(n) \rangle_{\Theta,\tau} $')
    plt.title('N ='+str(N)+' averaged over '+str(K)+' configurations')
    fig.savefig('N_'+str(N)+'_conf_'+str(K)+'_T_'+str(T)+'.png')
    return(time_avg)

if __name__ == '__main__':
    N = int(sys.argv[2])
    K = int(sys.argv[4])
    T = int(sys.argv[6])
    #population_data = dict()
    tSteps = int(sys.argv[8])
    params = (K,T,tSteps)
    print('works!',N,params)
    data = main(N,params)
    print(data)

reason: import numpy,from numpy,from scipy


text:
from ..core.proteins import ProteinStructure
from ..core.sequences import KinaseDomainAminoAcidSequence
from typing import Union

class Alignment:
    """Alignment representation of protein sequences"""

    def __init__(self, metadata=None, alignment=None, alignment_file_path=None, *args, **kwargs):

        from appdirs import user_cache_dir

        if metadata is None:
            metadata = {}
        self.metadata = metadata
        self.alignment = alignment
        self.alignment_file_path = f"{user_cache_dir()}/alignment.ali"

    @classmethod
    def get_alignment(cls, seq1: str, seq2: str, local: bool = True):
        """
        Generate an alignment between two sequences

        Parameters
        ----------
        seq1: str
            The first sequence to be aligned
        seq1: str
            The second sequence to be aligned
        local: bool
            If false, a global alignment is performed
            (based on the Needleman-Wunsch algorithm),
            otherwise a local alignment is performed
            (based on the Smith–Waterman algorithm).
            (Default: True)

        Returns
        -------
        Alignment
        """

        import biotite.sequence as seq
        import biotite.sequence.align as align
        import numpy as np

        # create the default matrix
        # TODO add more options for the choice of matrix
        matrix = align.SubstitutionMatrix.std_protein_matrix()

        alignments = align.align_optimal(
            seq.ProteinSequence(seq1),
            seq.ProteinSequence(seq2),
            matrix,
            local=local,
        )

        alignment = alignments[0]

        score = alignment.score
        seq_identity = align.get_sequence_identity(alignment)
        symbols = align.get_symbols(alignment)
        codes = align.get_codes(alignment)

        return cls(
            alignment=alignment,
            metadata={
                "score": score,
                "sequence_identity": seq_identity,
                "symbols": symbols,
                "codes": codes,
            },
        )

    def _format_alignment_with_ligands(self, template, aligned_template_seq):

        from MDAnalysis.lib.util import convert_aa_code

        sel = template.universe.select_atoms("protein and name CA")
        resnames = sel.resnames.tolist()
        resnames_one_letter = [convert_aa_code(x) for x in resnames]

        # get the last four AAs in the aligned target sequence
        end_chunk = aligned_template_seq[-4:][::-1]
        aa_store = []
        # loop over residues in reverse
        for i, letter in enumerate(resnames_one_letter[::-1]):
            # read a chunk of four AAs
            current_chunk = resnames_one_letter[::-1][i : i + 4]

            # if the end of the aligned target sequence is hit, stop
            if current_chunk == end_chunk:
                break
            # otherwise store the one letter AAs that need to be added
            else:
                aa_store.append(letter)

        # create a one element list of dashes
        dashes = "".join(["-" for x in aa_store])
        # create one element list of AAs in the canonical order
        amino_acids = "".join(aa_store[::-1])

        return amino_acids, dashes

    def make_ali_file(
        self,
        aligned_template_seq: str,
        aligned_target_seq: str,
        template: ProteinStructure,
        target: Union[str, KinaseDomainAminoAcidSequence],
        ligand: bool = False,
    ):
        """
        Generate an alignment file in MODELLER format

        Parameters
        ----------
        aligned_template_seq: str
            The aligned template sequence
        aligned_target_seq: str
            The aligned target sequence
        template: ProteinStructure
            The template structure
        target: list of str or KinaseDomainAminoAcidSequence
            The original target sequence
        ligand: bool
            Specify whether to retain a ligand in the alignment.
            (Default: False)
        """

        # Convert None entries into dashes
        seq1_dashed = [i or "-" for i in aligned_template_seq]
        seq2_dashed = [i or "-" for i in aligned_target_seq]

        # Setup formatting for MODELLER alignment file
        max_length = 75

        # TODO handle if using backend='ncbi'

        # handle if target is KinaseDomainAminoAcidSequence vs. str
        try:
            sequence_id = getattr(target, "metadata")["uniprot_id"]
            sequence_begin = getattr(target, "metadata")["begin"]
            sequence_end = getattr(target, "metadata")["end"]
        except:
            sequence_id = "sequence_id"
            sequence_begin = "1"
            sequence_end = len(target)

        protein_id = template.metadata["id"]
        # get index for the first residue in the structure sequence
        # ignoring negative and 0 resids
        resids = template.universe.residues.resids
        protein_start_index = next(
            (i for i, x in enumerate(template.universe.residues.resids) if x > 0),
            None,
        )
        protein_start = resids[protein_start_index]
        protein_end = ""

        if ligand:
            # extract the AAs and No. dashes that need to be added in the alignment
            tail_amino_acids, tail_dashes = self._format_alignment_with_ligands(
                template, aligned_template_seq
            )

            # add on bufffer AAs / dashes so that Modeller finds the ligand correctly
            seq1_dashed += tail_amino_acids
            seq2_dashed += tail_dashes

        # write alignment file in MODELLER format
        with open(self.alignment_file_path, "w") as ali_file:
            for i in range(len(seq1_dashed)):
                if i == 0:
                    ali_file.write(f">P1;{protein_id}\n")
                    ali_file.write(
                        f"structure:{protein_id}:{protein_start}:A:{protein_end}: :::     :     \n"
                    )
                ali_file.write(seq1_dashed[i])
                if i == len(seq1_dashed) - 1:
                    if ligand:
                        ali_file.write("/.*")
                    else:
                        ali_file.write("*")
                if (i + 1) % max_length == 0:
                    ali_file.write("\n")

            for i in range(len(seq2_dashed)):
                # start new line below first sequence
                if i == 0:
                    ali_file.write(f"\n>P1;{sequence_id}\n")
                    if ligand:
                        ali_file.write(
                            f"sequence:{sequence_id}:{sequence_begin}: :{sequence_end + len(tail_dashes)}: :::     :     \n"
                        )
                    else:
                        ali_file.write(
                            f"sequence:{sequence_id}:{sequence_begin}: :{sequence_end}: :::     :     \n"
                        )
                ali_file.write(seq2_dashed[i])
                if i == len(seq2_dashed) - 1:
                    if ligand:
                        ali_file.write("/.*")
                    else:
                        ali_file.write("*")
                if (i + 1) % max_length == 0:
                    ali_file.write("\n")

max_stars_repo_name: t-kimber/kinoml

max_issues_repo_name: t-kimber/kinoml

max_forks_repo_name: t-kimber/kinoml

reason: import numpy


text:
"""An example of how to use your own dataset to train a classifier that recognizes people.
"""
# MIT License
#
# Copyright (c) 2016 David Sandberg
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf
import numpy as np
import argparse
import facenet
import os
import sys
import math
import pickle
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier as RFC
import time
import multiprocessing
from pdb import set_trace as bp
import functools

def ms1m_name_list_reader(fname):
    assert os.path.isfile(fname), "{} doesn't exist".format(fname)

    result_dict = {}
    with open(fname, "r") as fin:
        for line in fin.readlines():
            msid, name = line.strip().split("\t")
            if not msid in result_dict:
                result_dict[msid] = [name]
            else:
                result_dict[msid].append(name)

    return result_dict

def do_work( emb_array, result_dict_predictions, result_dict_class_names, classifier_filename_exp, subset_model_and_class_names):
                        # gw: todo, find a way to append matrices to the right of existing ones,

    # Classify images
    subset_model, subset_class_names = subset_model_and_class_names
    begin = time.perf_counter()
    subset_predictions = subset_model.predict_proba(emb_array)
    end = time.perf_counter()

    print('classified subset {} images in {} seconds'.format(len(emb_array), end-begin))

    result_dict_predictions[classifier_filename_exp] = subset_predictions
    result_dict_class_names[classifier_filename_exp] = subset_class_names[:]

def str_eq(a,b):
    return str(a) == str(b)

arr_eq = np.frompyfunc(str_eq, 2,1)

# def get_arr_val(arr, idx):
#     return arr[idx]

# arr_get_arr_val = np.frompyfunc(get_arr_val, 2, 1)

def main(args):

    # init
    # load subset classifiers
    classifier_filename_exp_list = [os.path.expanduser(fname) for fname in args.classifier_filename]
    subset_model_and_class_names_list = []
    for classifier_filename_exp in classifier_filename_exp_list:
        start = time.perf_counter()
        with open(classifier_filename_exp, 'rb') as infile:
            (subset_model, subset_class_names, _) = pickle.load(infile)
            subset_model_and_class_names_list.append((subset_model, subset_class_names))
        end = time.perf_counter()

        print('Loaded classifier subset_model from file {} in {} seconds'.format(classifier_filename_exp, end - start))

    predictions = None
    combined_class_names = None
    mgr = multiprocessing.Manager()
    result_dict_predictions = mgr.dict()
    result_dict_class_names = mgr.dict()

    # load msid to human-readable name mapping
    msid_to_name_dict = ms1m_name_list_reader("/home/gaopeng/workspace/ms1m-aligned-full/Top1M_MidList.Name.tsv")

    # gw: bookmark 1129 afternoon
    # ----done init----

    with tf.Graph().as_default():

        with tf.Session() as sess:

            np.random.seed(seed=args.seed)

            # input loop for classification
            while True:

                # e.g.: /home/gaopeng/workspace/ms1m-aligned-full/gw_celeb_3500_20_val/
                data_dir = input("Input a data dir for classification: ")
                dataset = facenet.get_dataset(data_dir)

                # Check that there are at least one training image per class
                for cls in dataset:
                    assert(len(cls.image_paths)>0, 'There must be at least one image for each class in the dataset')

                paths, label_numbers, label_names = facenet.get_image_paths_and_labelnames(dataset)

                # label_names = [ name.replace('_', ' ') for name in label_names]

                print('Number of classes: %d' % len(dataset))
                print('Number of images: %d' % len(paths))

                # Load the subset_model
                print('Loading feature extraction subset_model')
                facenet.load_model(args.model)

                # Get input and output tensors
                images_placeholder = tf.get_default_graph().get_tensor_by_name("input:0")
                embeddings = tf.get_default_graph().get_tensor_by_name("embeddings:0")
                phase_train_placeholder = tf.get_default_graph().get_tensor_by_name("phase_train:0")
                embedding_size = embeddings.get_shape()[1]

                # Run forward pass to calculate embeddings
                print('Calculating features for images')
                nrof_images = len(paths)
                nrof_batches_per_epoch = int(math.ceil(1.0*nrof_images / args.batch_size))
                emb_array = np.zeros((nrof_images, embedding_size))
                for i in range(nrof_batches_per_epoch):
                    start_index = i*args.batch_size
                    end_index = min((i+1)*args.batch_size, nrof_images)
                    paths_batch = paths[start_index:end_index]
                    images = facenet.load_data(paths_batch, False, False, args.image_size)
                    feed_dict = { images_placeholder:images, phase_train_placeholder:False }
                    emb_array[start_index:end_index,:] = sess.run(embeddings, feed_dict=feed_dict)

                #------------- classifiying
                print('Testing classifier')

                # done: extracted out one-time expensive task out as initialization (model loading)
                # todo: how to make it a long running process?
                jobs = [multiprocessing.Process(target=do_work, args=(emb_array, result_dict_predictions, result_dict_class_names, classifier_filename_exp_list[i], subset_model_and_class_names_list[i])) for i in range(8)]

                for job in jobs:
                    job.start()

                for job in jobs:
                    job.join()

                #bp()
                for classifier_filename_exp in classifier_filename_exp_list:
                    subset_prediction = result_dict_predictions[classifier_filename_exp]
                    subset_class_names = result_dict_class_names[classifier_filename_exp]

                    if predictions is None:
                        predictions = subset_prediction
                        combined_class_names = subset_class_names[:]
                        #bp()

                    else:
                        predictions = np.concatenate((predictions, subset_prediction), axis=1)
                        combined_class_names = np.concatenate((combined_class_names, subset_class_names[:]), axis=0)

                # bp()

                best_class_indices = np.argmax(predictions, axis=1)
                best_class_names = np.array([combined_class_names[idx] for idx in best_class_indices])
                #best_class_names = np.arang;  # gw: todo: use ix_ to make a ndarray with row being test_indices, and col (only 1) being the best class name
                # best_class_indices = np.argsort(predictions, axis=1)[-8:] #gw: todo, calcuate embedding distance again amont 8 rfc results
                best_class_probabilities = predictions[np.arange(len(best_class_indices)), best_class_indices]

                def show_human_name_or_raw_class_name(msid):
                    if msid in msid_to_name_dict:
                        for name in msid_to_name_dict[msid]:
                            if "@en" in name:
                                return name
                            elif "@zh" in name:
                                return name
                            elif "@zh-Hant" in name:
                                return name

                    return msid

                for i in range(len(best_class_indices)):
                    print('actual name (msid): %s (%s),\t, pred name (msid): %s (%s), \t prob:  %.3f, \t img: %s' % (show_human_name_or_raw_class_name(label_names[i]),
                                                                                label_names[i],
                                                                                show_human_name_or_raw_class_name(combined_class_names[best_class_indices[i]]),
                                                                                combined_class_names[best_class_indices[i]],
                                                                                best_class_probabilities[i],
                                                                                paths[i]))

                # accuracy = np.mean(np.equal(best_class_indices, label_numbers))
                # accuracy = np.mean(np.equal(best_class_indices, label_names))
                # bp()
                accuracy = np.mean(arr_eq(best_class_names, label_names))
                print('Accuracy: %.13f' % accuracy)

def split_dataset(dataset, min_nrof_images_per_class, nrof_train_images_per_class):
    train_set = []
    test_set = []
    for cls in dataset:
        paths = cls.image_paths
        # Remove classes with less than min_nrof_images_per_class
        if len(paths)>=min_nrof_images_per_class:
            np.random.shuffle(paths)
            train_set.append(facenet.ImageClass(cls.name, paths[:nrof_train_images_per_class]))
            test_set.append(facenet.ImageClass(cls.name, paths[nrof_train_images_per_class:]))
    return train_set, test_set

# python src/classifier_multi.py CLASSIFY --data_dir /home/gaopeng/workspace/ms1m-aligned-full/gw_celeb_3500_20_val/ --subset_model /home/gaopeng/models/facenet/20180402-114759/20180402-114759.pb --classifier_filename ~/models/gw_subset_{1..9}_rfc_classifier.pkl --batch_size 1536 --min_nrof_images_per_class 20 --nrof_train_images_per_class 15 --use_split_dataset
def parse_arguments(argv):
    parser = argparse.ArgumentParser()

    parser.add_argument('--model', type=str,
        help='Could be either a directory containing the meta_file and ckpt_file or a model protobuf (.pb) file')
    parser.add_argument('--classifier_filename',
                        nargs='*',
        help='Classifier model file name as a pickle (.pkl) file. ' +
        'For training this is the output and for classification this is an input.')
    parser.add_argument('--test_data_dir', type=str,
        help='Path to the test data directory containing aligned images used for testing.')
    parser.add_argument('--batch_size', type=int,
        help='Number of images to process in a batch.', default=90)
    parser.add_argument('--image_size', type=int,
        help='Image size (height, width) in pixels.', default=160)
    parser.add_argument('--seed', type=int,
        help='Random seed.', default=666)
    parser.add_argument('--min_nrof_images_per_class', type=int,
        help='Only include classes with at least this number of images in the dataset', default=20)
    parser.add_argument('--nrof_train_images_per_class', type=int,
        help='Use this number of images from each class for training and the rest for testing', default=10)

    return parser.parse_args(argv)

if __name__ == '__main__':

    main(parse_arguments(sys.argv[1:]))
    # sample invocation:
    # (venv_facenet) gaopeng@gw-cm6870:~/workspace/python3.6/facenet$ python src/classifier_daemon_subsets_with_multiproc.py --model /home/gaopeng/models/facenet/20180402-114759/20180402-114759.pb --classifier_filename ~/models/gw_subset_{1..8}_svc_classifier.pkl --batch_size 1536 --min_nrof_images_per_class 20 --nrof_train_images_per_class 15


text:
#!/usr/bin/env python

# import time
import os
import numpy as np

from pycrazyswarm import *
# from crazyflie import *
# from sensor_msgs.msg import Joy

DY = 0.4 #m
DX = 0.3 #m
DZ = 0.3 #m
OX = 1
OY = 0.66
OZ = 0.5
DURATION = 1.4
SLEEP = 0.5
TRIALS = 100

# wantToExit = False

def hover(cf, pos, duration):
    cf.hover(
        np.array(pos) * np.array([DX, DY, DZ])
        + np.array([OX, OY, OZ])
        , 0, duration)

# def joyChanged(data):
#     global wantToExit
#     if data.buttons[5] == 1:
#         wantToExit = True

def sleep(duration):
    start = timeHelper.time()
    while True:
        now = timeHelper.time()
        elapsed = now - start
        if elapsed > duration:
            break
        if swarm.input.checkIfButtonIsPressed():
            for cf in allcfs.crazyflies:
                cf.hover(np.array(cf.initialPosition) + np.array([0, 0, 1]), 0, 3.0)
            timeHelper.sleep(4.0)
            allcfs.land(targetHeight = 0.04, duration = 2.0)
            timeHelper.sleep(2.0)
            os._exit(0)
        timeHelper.sleep(0.01)

if __name__ == "__main__":
    swarm = Crazyswarm()
    timeHelper = swarm.timeHelper
    allcfs = swarm.allcfs

    # print("wait")
    # swarm.input.waitUntilButtonPressed()

    # allcfs = CrazyflieServer()
    # rospy.Subscriber("/joy", Joy, joyChanged)

    allcfs.takeoff(targetHeight=1.0, duration=2.0)
    sleep(2.0)

    cfs = allcfs.crazyflies

    for i in range(0, TRIALS):

        hover(cfs[0], [0, 0, 5], DURATION)
        hover(cfs[1], [0, -3, 5], DURATION)
        hover(cfs[2], [-1, 0, 4], DURATION)
        hover(cfs[3], [-1, -3, 4], DURATION)
        hover(cfs[4], [-2, 0, 3], DURATION)
        hover(cfs[5], [-2, -3, 3], DURATION)
        hover(cfs[6], [-3, 0, 2], DURATION)
        hover(cfs[7], [-3, -1, 1], DURATION)
        hover(cfs[8], [-3, -2, 1], DURATION)
        hover(cfs[9], [-3, -3, 2], DURATION)

        sleep(DURATION + SLEEP)

        hover(cfs[0], [0, -1, 5], DURATION)
        # hover(cfs[1], [0, -3, 5], DURATION)
        # hover(cfs[2], [-1, 0, 4], DURATION)
        hover(cfs[3], [0, -2, 5], DURATION)
        hover(cfs[4], [-2, -1, 3], DURATION)
        hover(cfs[5], [-2, -2, 3], DURATION)
        hover(cfs[6], [-3, 0, 1], DURATION)
        # hover(cfs[7], [-3, -1, 1], DURATION)
        # hover(cfs[8], [-3, -2, 1], DURATION)
        # hover(cfs[9], [-3, -3, 2], DURATION)

        sleep(DURATION + SLEEP)

        hover(cfs[0], [0, -0.75, 5], DURATION)
        hover(cfs[1], [0, -2.25, 5], DURATION)
        # hover(cfs[2], [-1, 0, 4], DURATION)
        hover(cfs[3], [0, -1.5, 5], DURATION)
        hover(cfs[4], [-2, 0, 3], DURATION)
        hover(cfs[5], [-1, -3, 5], DURATION)
        hover(cfs[6], [-3, 0, 2], DURATION)
        # hover(cfs[7], [-3, -1, 1], DURATION)
        # hover(cfs[8], [-3, -2, 1], DURATION)
        hover(cfs[9], [-3, -3, 1], DURATION)

        sleep(DURATION + SLEEP)

# 0    1
# 2    3
# 4    5
# 6    9
#  7 8

#   0 3 1
# 2
#   4 5
#       9
# 6 7 8

#   0 3 1
# 2       5
# 4
# 6
#   7 8 9

max_stars_repo_name: hugokohli/SwarmInterface

max_issues_repo_name: TaoChenOSU/crazyswarm-nn

max_forks_repo_name: TaoChenOSU/crazyswarm-nn


text:
# Copyright 2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# less required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""train callbacks"""
try:
    from moxing.framework import file
    print("import moxing success")
except ModuleNotFoundError as e:
    print(f'not modelarts env, error={e}')

import os
import time

import numpy as np

from mindspore.common.tensor import Tensor
import mindspore.common.dtype as mstype
from mindspore.train.serialization import save_checkpoint
from mindspore.train.callback._callback import Callback

class LossMonitor(Callback):
    """
    Monitor the loss in training.

    If the loss is NAN or INF, it will terminate training.

    Note:
        If per_print_times is 0, do not print loss.

    Args:
        per_print_times (int): Print the loss each every time. Default: 1.

    Raises:
        ValueError: If print_step is not an integer or less than zero.
    """

    def __init__(self, per_print_times=1):
        super(LossMonitor, self).__init__()
        if not isinstance(per_print_times, int) or per_print_times < 0:
            raise ValueError("print_step must be int and >= 0.")
        self._per_print_times = per_print_times

    def step_end(self, run_context):
        """
        Call at each step end.
        """
        cb_params = run_context.original_args()
        loss = cb_params.net_outputs

        if isinstance(loss, (tuple, list)):
            if isinstance(loss[0], Tensor) and isinstance(loss[0].asnumpy(), np.ndarray):
                loss = loss[0]

        if isinstance(loss, Tensor) and isinstance(loss.asnumpy(), np.ndarray):
            loss = np.mean(loss.asnumpy())

        cur_step_in_epoch = (cb_params.cur_step_num -
                             1) % cb_params.batch_num + 1

        if isinstance(loss, float) and (np.isnan(loss) or np.isinf(loss)):
            raise ValueError(
                "step: {}. Invalid loss, terminating training.".format(cur_step_in_epoch))
        if self._per_print_times != 0 and cb_params.cur_step_num % self._per_print_times == 0:
            print("step: %s, loss is %s" %
                  (cur_step_in_epoch, loss), flush=True)

class Val_Callback(Callback):
    """
    Valid the test data at every epoch.
    """

    def __init__(self, model, train_dataset, val_dataset, checkpoint_path, prefix,
                 network, img_size, device_id=0, is_eval_train_dataset='False'):
        super(Val_Callback, self).__init__()
        self.model = model
        self.train_dataset = train_dataset
        self.val_dataset = val_dataset
        self.checkpoint_path = checkpoint_path
        self.max_val_acc = 0
        self.prefix = prefix
        self.network = network
        self.img_size = img_size
        self.device_id = device_id
        self.is_eval_train_dataset = is_eval_train_dataset

    def epoch_end(self, run_context):
        """
        Call at each epoch end.
        """
        if self.is_eval_train_dataset == 'True':
            start = time.time()
            train_result = self.model.eval(
                self.train_dataset, dataset_sink_mode=False)
            end = time.time()
            print("==========train metrics:" + str(train_result) + " use times:" +
                  str((end - start) * 1000) + "ms=========================")
        start = time.time()
        val_result = self.model.eval(self.val_dataset, dataset_sink_mode=False)
        end = time.time()
        print("==========val metrics:" + str(val_result) + " use times:" +
              str((end - start) * 1000) + "ms=========================")
        val_acc = val_result['top_1_accuracy']
        if val_acc > self.max_val_acc:
            print('=================save checkpoint....====================')
            self.max_val_acc = val_acc
            cb_params = run_context.original_args()
            epoch = cb_params.cur_epoch_num
            model_info = self.prefix + '_id' + str(self.device_id) + \
                '_epoch' + str(epoch) + '_valacc' + str(val_acc)
            if self.checkpoint_path.startswith('s3://') or self.checkpoint_path.startswith('obs://'):
                save_path = '/cache/save_model/'
            else:
                save_path = self.checkpoint_path
            save_path = os.path.join(save_path, model_info)
            # save checkpoint
            ckpt_path = os.path.join(save_path, 'checkpoint')
            if not os.path.exists(ckpt_path):
                os.makedirs(ckpt_path)
            ckpt_file = os.path.join(ckpt_path, 'model_checkpoint.ckpt')
            save_checkpoint(cb_params.train_network, ckpt_file)
            if self.checkpoint_path.startswith('s3://') or self.checkpoint_path.startswith('obs://'):
                file.copy_parallel(save_path, os.path.join(
                    self.checkpoint_path, model_info))
            print('==============save checkpoint finished===================')
        print(f'The best accuracy is {self.max_val_acc}')

class Set_Attr_CallBack(Callback):
    """
    Set drop_path_prob and epoch_mask.

    """

    def __init__(self, model, drop_path_prob, epochs, layers, batch_size):
        super(Set_Attr_CallBack, self).__init__()
        self.model = model
        self.drop_path_prob = drop_path_prob
        self.epochs = epochs
        self.layers = layers
        self.batch_size = batch_size

    def epoch_begin(self, run_context):
        """
        Call at each epoch begin.
        """
        self.model.drop_path_prob = self.drop_path_prob * 300 / self.epochs
        keep_prob = 1. - self.model.drop_path_prob
        self.model.epoch_mask = []
        for _ in range(self.layers):
            layer_mask = []
            for _ in range(5 * 2):
                mask = np.array([np.random.binomial(1, p=keep_prob)
                                 for k in range(self.batch_size)])
                mask = mask[:, np.newaxis, np.newaxis, np.newaxis]
                mask = Tensor(mask, mstype.float16)
                layer_mask.append(mask)
            self.model.epoch_mask.append(layer_mask)

max_stars_repo_name: mindspore-ai/models

max_issues_repo_name: mindspore-ai/models

max_forks_repo_name: mindspore-ai/models


text:
import pandas as pd
import numpy as np
from scipy.optimize import minimize
from sklearn.cross_validation import StratifiedShuffleSplit
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import log_loss
import os

os.system("ls ../input")

train = pd.read_csv("../input/train.csv")
print(("Training set has {0[0]} rows and {0[1]} columns".format(train.shape)))

labels = train['target']
train.drop(['target', 'id'], axis=1, inplace=True)

print((train.head()))

### we need a test set that we didn't train on to find the best weights for combining the classifiers
sss = StratifiedShuffleSplit(labels, test_size=0.05, random_state=1234)
for train_index, test_index in sss:
    break

train_x, train_y = train.values[train_index], labels.values[train_index]
test_x, test_y = train.values[test_index], labels.values[test_index]

### building the classifiers
clfs = []

rfc = RandomForestClassifier(n_estimators=50, random_state=4141, n_jobs=-1)
rfc.fit(train_x, train_y)
print(('RFC LogLoss {score}'.format(score=log_loss(test_y, rfc.predict_proba(test_x)))))
clfs.append(rfc)

### usually you'd use xgboost and neural nets here

logreg = LogisticRegression()
logreg.fit(train_x, train_y)
print(('LogisticRegression LogLoss {score}'.format(score=log_loss(test_y, logreg.predict_proba(test_x)))))
clfs.append(logreg)

rfc2 = RandomForestClassifier(n_estimators=50, random_state=1337, n_jobs=-1)
rfc2.fit(train_x, train_y)
print(('RFC2 LogLoss {score}'.format(score=log_loss(test_y, rfc2.predict_proba(test_x)))))
clfs.append(rfc2)

### finding the optimum weights

predictions = []
for clf in clfs:
    predictions.append(clf.predict_proba(test_x))

def log_loss_func(weights):
    ''' scipy minimize will pass the weights as a numpy array '''
    final_prediction = 0
    for weight, prediction in zip(weights, predictions):
            final_prediction += weight*prediction

    return log_loss(test_y, final_prediction)

#the algorithms need a starting value, right not we chose 0.5 for all weights
#its better to choose many random starting points and run minimize a few times
starting_values = [0.5]*len(predictions)

#adding constraints  and a different solver as suggested by user 16universe
#https://kaggle2.blob.core.windows.net/forum-message-attachments/75655/2393/otto%20model%20weights.pdf?sv=2012-02-12&se=2015-05-03T21%3A22%3A17Z&sr=b&sp=r&sig=rkeA7EJC%2BiQ%2FJ%2BcMpcA4lYQLFh6ubNqs2XAkGtFsAv0%3D
cons = ({'type':'eq','fun':lambda w: 1-sum(w)})
#our weights are bound between 0 and 1
bounds = [(0,1)]*len(predictions)

res = minimize(log_loss_func, starting_values, method='SLSQP', bounds=bounds, constraints=cons)

print(('Ensamble Score: {best_score}'.format(best_score=res['fun'])))
print(('Best Weights: {weights}'.format(weights=res['x'])))

max_stars_repo_path: data/external/repositories_2to3/160758/kaggle_otto-master/7 - Ensemble (sample).py

max_stars_repo_name: Keesiu/meta-kaggle

max_issues_repo_name: Keesiu/meta-kaggle

max_forks_repo_name: Keesiu/meta-kaggle

reason: import numpy,from scipy


text:
#Naive Bayes Classifier using Numpy
import numpy as np

class NaiveBayes:

    def fit(self, X, y):
    """Function to establish relation between dependent and independent variable
    Args:
        X: Independent variable
        y: Dependent variable
    Returns:
        None
    """
        n_samples, n_features = X.shape
        self._classes = np.unique(y)
        n_classes = len(self._classes)

        self._mean = np.zeros((n_classes, n_features), dtype= np.float64)
        self._var = np.zeros((n_classes, n_features), dtype= np.float64)
        self._priors = np.zeros(n_classes, dtype = np.float64)

        for c in self._classes:
            X_c = X[c==y]
            self._mean[c,:] = X_c.mean(axis=0)
            self._var[c,:] = X_c.var(axis=0)
            self.priors[c] = X_c.shape[0] / float(n_samples)

    def predict(self, X):
        y_pred = [self._predict(x) for x in X]
        return y_pred

    def _predict(self, x):
        posteriors = []

        for idx, c in enumerate(self._classes):
            prior = np.log(self._priors[idx])
            class_conditional = np.sum(np.log(self._pdf(idx, x)))
            posterior = prior + class_conditional
            posteriors.append(posterior)

        return self._classes[np.argmax(posteriors)]

    def _pdf(self, class_idx, x):
        mean = self._mean[class_idx]
        var = self._var[class_idx]
        numerator = np.exp(- (x - mean) ** 2 / ( 2 * var ))
        denominator = np.sqrt(2*np.pi*var)
        return (numerator / denominator)

max_stars_repo_name: VallabhK/MachineLearning

max_issues_repo_name: VallabhK/MachineLearning

max_forks_repo_name: VallabhK/MachineLearning


text:
#!/usr/bin/env python
# -*- coding: UTF-8 -*-

import torch
import torch.nn as nn
import torch.nn.functional as F
#import math
#import numpy as np
import Submodules
import logging
logger = logging.getLogger(__name__)

class WSMCnetBase(nn.Module):

    def __init__(self, name_model, maxdisp):
        super(WSMCnetBase, self).__init__()

        # maximum of disparity
        self.maxdisp = maxdisp

        # parse S, C and F3 from name_model
        S, C, F3 = self._postfix_parse(name_model)
        self.S = S
        self.C = C
        self.F3 = F3

        # modules of [feature_extraction and cost_compute]
        self.feature_extraction = None
        self.cost_compute = None

    @property
    def name(self):
        return self._get_name() + self.postfix

    @property
    def postfix(self):
        return '_S%dC%dF%d'%(self.S, self.C, self.F3)

    def _postfix_parse(self, name_model):

        S, C, F3 = 1, 1, 32
        strs = name_model.lower().split('_')
        if(2 == len(strs)):
            subfix = strs[1]
            idxs = [subfix.find(tmp) for tmp in ['s', 'c', 'f']]
            if(idxs[1] > 0):
                S = int(subfix[idxs[0]+1:idxs[1]])
                if(idxs[2] > 0):
                    C = int(subfix[idxs[1]+1:idxs[2]])
                    F3 = int(subfix[idxs[2]+1:])
                else:
                    C = int(subfix[idxs[1]+1:])
            else:
                S = int(subfix[idxs[0]+1:])
            S = max(1, min(16, S))
            C = max(1, min(16, C))
            F3 = max(8, min(64, F3))

        return (S, C, F3)

    def get_matching_costs(self, left, right):

        # feature extraction
        bn = left.shape[0]
        features = torch.cat([left, right], dim=0)
        features = self.feature_extraction(features)
        fL, fR = features[:bn], features[bn:]

        # compute matching cost
        stride = self.S
        shift = (self.maxdisp//(stride<<2)) + 1
        cost = Submodules.cost_volume_gen(fL, fR, shift, stride)
        costs = self.cost_compute(cost)

        return costs

    def cost_disps(self, disps, disp_true):

        # split disp
        output1, output2, output3 = disps

        # mask of validate labels
        maxdisp = self.maxdisp
        mask =  (disp_true > 0) & (disp_true < maxdisp)

        # compute loss
        lossfun = F.smooth_l1_loss
        loss = 0.5*lossfun(output1[mask], disp_true[mask], size_average=True) + \
                0.7*lossfun(output2[mask], disp_true[mask], size_average=True) + \
                lossfun(output3[mask], disp_true[mask], size_average=True)

        return loss

    def cost_similarity(self, similarities, disp_true):

        # split disp
        output1, output2, output3 = similarities

        # mask of validate labels
        maxdisp = self.maxdisp
        mask_invalid =  ~((disp_true > 0) & (disp_true < maxdisp))
        disp_true[mask_invalid] = float('inf')

        # compute loss
        lossfun = Submodules.subpixel_cross_entropy_loss
        loss = 0.5*lossfun(output1, disp_true, self.disp_step) + \
                0.7*lossfun(output2, disp_true, self.disp_step) + \
                lossfun(output3, disp_true, self.disp_step)

        return loss

    def cost_similarity_disps(self, output, disp_true):

        # split output
        similarities, disps = output

        # L1 loss
        loss_L1 = self.cost_disps(disps, disp_true)

        # crose-entropy loss
        loss_ce = self.cost_similarity(similarities, disp_true)

        if(loss_L1<10):
            return loss_ce + 0.2*loss_L1
        else:
            return loss_ce

class WSMCnetTri_L1(WSMCnetBase):

    def __init__(self, name_model, maxdisp):
        super(WSMCnetTri_L1, self).__init__(name_model, maxdisp)

        # upsample and softmax
        self.upsample = nn.Upsample(scale_factor=4, mode='trilinear', align_corners=True)
        self.disp_step = float(self.S)/self.C

    def forward(self, left, right):

        # compute matching costs
        costs = self.get_matching_costs(left, right)

        # disparty regression
        h, w = left.shape[-2:]
        disps = []
        for tcost in costs:
            tcost = self.upsample(tcost).squeeze(1)[..., :h, :w]
            disps.append(Submodules.disp_regression(tcost, self.disp_step))

        # return
        logger.debug('training: %s \n' % str(self.training))
        return disps if self.training else disps[-1]

    def compute_cost(self, disps, disp_true):
        return self.cost_disps(disps, disp_true)

class WSMCnetBi_L1(WSMCnetBase):

    def __init__(self, name_model, maxdisp):
        super(WSMCnetBi_L1, self).__init__(name_model, maxdisp)

        # upsample and softmax
        self.upsample = nn.Upsample(scale_factor=4, mode='bilinear', align_corners=True)
        self.disp_step = float(self.S<<2)/self.C

    def forward(self, left, right):

        # compute matching costs
        costs = self.get_matching_costs(left, right)

        # disparty regression
        h, w = left.shape[-2:]
        disps = []
        for tcost in costs:
            tcost = self.upsample(tcost.squeeze(1))[..., :h, :w]
            disps.append(Submodules.disp_regression(tcost, self.disp_step))

        # return
        logger.debug('training: %s \n' % str(self.training))
        return disps if self.training else disps[-1]

    def compute_cost(self, disps, disp_true):
        return self.cost_disps(disps, disp_true)

class WSMCnet(WSMCnetTri_L1):

    def __init__(self, name_model, maxdisp):
        super(WSMCnet, self).__init__(name_model, maxdisp)

        # modules of [feature_extraction and cost_compute]
        from Submodules.FeatureExtraction import FeatureExtraction
        from Submodules.CostCompute import CostCompute
        self.feature_extraction = FeatureExtraction(planes=32)
        self.cost_compute = CostCompute(inplanes=64, planes=self.F3, C=self.C)

        # init weight
        Submodules.weight_init(self.modules())

class WSMCnetB(WSMCnetBi_L1):

    def __init__(self, name_model, maxdisp):
        super(WSMCnetB, self).__init__(name_model, maxdisp)

        # modules of [feature_extraction and cost_compute]
        from Submodules.FeatureExtraction import FeatureExtraction
        from Submodules.CostCompute import CostCompute
        self.feature_extraction = FeatureExtraction(planes=32)
        self.cost_compute = CostCompute(inplanes=64, planes=self.F3, C=self.C)

        # init weight
        Submodules.weight_init(self.modules())

class WSMCnetL(WSMCnetTri_L1):

    def __init__(self, name_model, maxdisp):
        super(WSMCnetL, self).__init__(name_model, maxdisp)

        # modules of [feature_extraction and cost_compute]
        from Submodules.FeatureExtraction import FeatureExtraction
        from Submodules.CostComputeL import CostCompute
        self.feature_extraction = FeatureExtraction(planes=32)
        self.cost_compute = CostCompute(inplanes=64, planes=self.F3, C=self.C)

        # init weight
        Submodules.weight_init(self.modules())

class WSMCnetLB(WSMCnetBi_L1):

    def __init__(self, name_model, maxdisp):
        super(WSMCnetLB, self).__init__(name_model, maxdisp)

        # modules of [feature_extraction and cost_compute]
        from Submodules.FeatureExtraction import FeatureExtraction
        from Submodules.CostComputeL import CostCompute
        self.feature_extraction = FeatureExtraction(planes=32)
        self.cost_compute = CostCompute(inplanes=64, planes=self.F3, C=self.C)

        # init weight
        Submodules.weight_init(self.modules())

class WSMCnetTri_CE(WSMCnetBase):

    def __init__(self, name_model, maxdisp):
        super(WSMCnetTri_CE, self).__init__(name_model, maxdisp)

        # upsample and softmax
        self.upsample = nn.Upsample(scale_factor=4, mode='trilinear', align_corners=True)
        self.disp_step = float(self.S)/self.C

    def forward(self, left, right):

        # compute matching costs
        costs = self.get_matching_costs(left, right)

        # disparty regression
        h, w = left.shape[-2:]
        costs = [self.upsample(tcost).squeeze(1)[..., :h, :w] for tcost in costs]
        disps = [Submodules.disp_regression_nearby(tcost, self.disp_step) for tcost in costs]

        if(self.training):
            return costs, disps
        else:
            return disps[-1]

    def compute_cost(self, output, disp_true):
        return self.cost_similarity_disps(output, disp_true)

class WSMCnetBi_CE(WSMCnetBase):

    def __init__(self, name_model, maxdisp):
        super(WSMCnetBi_CE, self).__init__(name_model, maxdisp)

        # upsample and softmax
        self.upsample = nn.Upsample(scale_factor=4, mode='bilinear', align_corners=True)
        self.disp_step = float(self.S<<2)/self.C

    def forward(self, left, right):

        # compute matching costs
        costs = self.get_matching_costs(left, right)

        # disparty regression
        h, w = left.shape[-2:]
        costs = [self.upsample(tcost.squeeze(1))[..., :h, :w] for tcost in costs]
        disps = [Submodules.disp_regression_nearby(tcost, self.disp_step) for tcost in costs]

        if(self.training):
            return costs, disps
        else:
            return disps[-1]

    def compute_cost(self, output, disp_true):
        return self.cost_similarity_disps(output, disp_true)

class WSMCnetE(WSMCnetTri_CE):

    def __init__(self, name_model, maxdisp):
        super(WSMCnetE, self).__init__(name_model, maxdisp)

        # modules of [feature_extraction and cost_compute]
        from Submodules.FeatureExtraction import FeatureExtraction
        from Submodules.CostCompute import CostCompute
        self.feature_extraction = FeatureExtraction(planes=32)
        self.cost_compute = CostCompute(inplanes=64, planes=self.F3, C=self.C)

        # init weight
        Submodules.weight_init(self.modules())

class WSMCnetEB(WSMCnetBi_CE):

    def __init__(self, name_model, maxdisp):
        super(WSMCnetEB, self).__init__(name_model, maxdisp)

        # modules of [feature_extraction and cost_compute]
        from Submodules.FeatureExtraction import FeatureExtraction
        from Submodules.CostCompute import CostCompute
        self.feature_extraction = FeatureExtraction(planes=32)
        self.cost_compute = CostCompute(inplanes=64, planes=self.F3, C=self.C)

        # init weight
        Submodules.weight_init(self.modules())

class WSMCnetEL(WSMCnetTri_CE):

    def __init__(self, name_model, maxdisp):
        super(WSMCnetEL, self).__init__(name_model, maxdisp)

        # modules of [feature_extraction and cost_compute]
        from Submodules.FeatureExtraction import FeatureExtraction
        from Submodules.CostComputeL import CostCompute
        self.feature_extraction = FeatureExtraction(planes=32)
        self.cost_compute = CostCompute(inplanes=64, planes=self.F3, C=self.C)

        # init weight
        Submodules.weight_init(self.modules())

class WSMCnetELB(WSMCnetBi_CE):

    def __init__(self, name_model, maxdisp):
        super(WSMCnetELB, self).__init__(name_model, maxdisp)

        # modules of [feature_extraction and cost_compute]
        from Submodules.FeatureExtraction import FeatureExtraction
        from Submodules.CostComputeL import CostCompute
        self.feature_extraction = FeatureExtraction(planes=32)
        self.cost_compute = CostCompute(inplanes=64, planes=self.F3, C=self.C)

        # init weight
        Submodules.weight_init(self.modules())

def get_model_by_name(name, maxdisp):

    name_class = name.split('_')[0]
    try:
        return eval(name_class)(name, maxdisp)
    except:
        raise Exception('Unsupported model: ' + name)

if __name__ == '__main__':

#    logging.basicConfig(level=logging.DEBUG, format=' %(asctime)s - %(levelname)s - %(message)s')
    logging.basicConfig(level=logging.INFO, format=' %(asctime)s - %(levelname)s - %(message)s')
    for name in [
                'WSMCnet', 'WSMCnetB', 'WSMCnetL', 'WSMCnetLB',
                'WSMCnetE', 'WSMCnetEB', 'WSMCnetEL', 'WSMCnetELB',
                ]:
        model = get_model_by_name(name + '_S2C3F32', 192)
        logger.info('%s passed!\n ' % model.name +
                    'Fun_active   : %s \n '% str(model.cost_compute.fun_active) +
                    'Fun_upsample : %s \n '% str(model.upsample) +
                    'disp_step : %s \n '% str(model.disp_step))


text:
# -*- coding: utf-8 -*-
import numpy as np
import torch
import itertools
from tqdm import tqdm

from utils.impvol import implied_volatility

from anderson_lake.integration import GaussianQuadrature
from anderson_lake.models import HestonModel
from anderson_lake.pricers import anderson_lake
from anderson_lake.options import EuropeanCallOption

# ==============================================================================
# === Functions
def scale_to_interval(x, a, b):
    xmin, xmax = np.min(x), np.max(x)
    return ( x - xmin ) / ( xmax - xmin ) * ( b - a ) + a

def generate_output_grid(x_min, x_max, y_min, y_max, num_dim):
    x_grid = np.linspace(x_min, x_max, num_dim[0])
    y_grid = np.linspace(y_min, y_max, num_dim[1])
    output_grid = np.array([x for x in itertools.product(x_grid, y_grid)])

    return output_grid

# ==============================================================================
# === Heston
def heston_closed_form(input_grid, output_grid):
    '''
    input_grid:     num_inputs x 7  (forward, vol, kappa, theta, sigma, rho, rate)
    output_grid:    num_outputs x 2 (tau, log-moneyness)
    '''

    m_impvol = np.ones((len(input_grid), len(output_grid))) * np.NaN
    m_prices = np.ones((len(input_grid), len(output_grid))) * np.NaN

    scheme = GaussianQuadrature(1e-8, 1e-8, 10000)
    for i, item_i in tqdm(enumerate(input_grid)):
        forward, vol, kappa, theta, sigma, rho, rate = item_i
        model = HestonModel(forward, vol, kappa, theta, sigma, rho, rate)

        for j, item_j in enumerate(output_grid):
            tau, log_moneyness = item_j
            strike = forward * np.exp(log_moneyness)
            option = EuropeanCallOption(tau, strike)

            # try calculating the implied volatility. If something fails
            # insert -1 instead.
            try:
                price = anderson_lake(model, option, scheme)
                impvol = implied_volatility(price, tau, forward, strike, rate)
            except:
                price = -1
                impvol = -1

            m_prices[i, j] = price
            m_impvol[i, j] = impvol

    return m_prices, m_impvol

def generate_heston_data(num_input, dim_output, skip=0):
    # boundaries for parameter space
    forward_min, forward_max = 75, 125
    vol_min, vol_max = 0.05, 0.15

    kappa_min, kappa_max = 1, 5
    theta_min, theta_max = 0.05, 0.15
    sigma_min, sigma_max = 0.2, 0.5
    rho_min, rho_max = -1.0, -0.7
    rate_min, rate_max = 0, 0.1

    # volatility surface grid
    tau_min, tau_max = 0.1, 0.5
    log_moneyness_min, log_moneyness_max = -0.25, 0.25

    sobol = torch.quasirandom.SobolEngine(7)
    sobol.fast_forward(skip)

    # generate input grid
    input_grid = np.array(sobol.draw(int(num_input)))

    min_v = [forward_min, vol_min, kappa_min, theta_min, sigma_min, rho_min, rate_min]
    max_v = [forward_max, vol_max, kappa_max, theta_max, sigma_max, rho_max, rate_max]

    for idx, (a, b) in enumerate(zip(min_v, max_v)):
        input_grid[:, idx] = scale_to_interval(input_grid[:, idx], a, b)

    # implied_volatility requires arguments to be of type float.
    # Apparently np.float32 is not considered a float.
    input_grid = input_grid.astype(np.float64)

    # generate output grid
    output_grid = generate_output_grid(tau_min, tau_max, \
                        log_moneyness_min, log_moneyness_max, dim_output)

    m_prices, m_impvol = heston_closed_form(input_grid, output_grid)

    return input_grid, output_grid, m_prices, m_impvol

if __name__ == "__main__":
    # To change the parameter bounds, see generate_heston_data.
    num_training_samples = 100000
    num_test_samples = 10000

    # Training set
    input_grid, output_grid, m_prices, m_impvol = \
        generate_heston_data(num_training_samples, (8, 11), skip=0)
    np.savetxt("training-data\input-grid-train.csv", input_grid)
    np.savetxt("training-data\prices-train.csv", m_prices)
    np.savetxt("training-data\impvol-train.csv", m_impvol)

    # Test set
    input_grid, output_grid, m_prices, m_impvol = \
        generate_heston_data(num_test_samples, (8, 11),
                             skip=num_training_samples)
    np.savetxt("training-data\input-grid-test.csv", input_grid)
    np.savetxt("training-data\prices-test.csv", m_prices)
    np.savetxt("training-data\impvol-test.csv", m_impvol)


text:
import cv2 as cv
import numpy as np

def find_slope(region,ker):
    # reg = np.array(region, dtype=np.float64)
    kern = np.array(ker, dtype=np.float64)
    divs = kern.shape[0]
    reg = np.ndarray([divs,region.shape[0],region.shape[1]], dtype=np.float64)
    reg[:] = np.array(region, dtype=np.float64)

    slope_max = np.amax((np.sum(np.multiply(reg,kern==255), axis=(1,2))/np.count_nonzero(kern==255, axis=(1,2)) - np.sum(np.multiply(reg,kern==0), axis=(1,2))/np.count_nonzero(kern==0, axis=(1,2)))/255)
    slope_min = np.amin((np.sum(np.multiply(reg,kern==255), axis=(1,2))/np.count_nonzero(kern==255, axis=(1,2)) - np.sum(np.multiply(reg,kern==0), axis=(1,2))/np.count_nonzero(kern==0, axis=(1,2)))/255)

    #slope_max + (slope_max-slope_min)
    # diff = 2*slope_max-slope_min
    diff = slope_max-slope_min
    max_slope_angle = np.argmax((np.sum(np.multiply(reg,kern==255), axis=(1,2))/np.count_nonzero(kern==255, axis=(1,2)) - np.sum(np.multiply(reg,kern==0), axis=(1,2))/np.count_nonzero(kern==0, axis=(1,2)))/255)

    max_diff_angle = np.argmax(((np.sum(np.multiply(reg,kern==255), axis=(1,2))/np.count_nonzero(kern==255, axis=(1,2)) - np.sum(np.multiply(reg,kern==0), axis=(1,2))/np.count_nonzero(kern==0, axis=(1,2)))/255) - ((np.sum(np.multiply(reg,kern==255), axis=(1,2))/np.count_nonzero(kern==255, axis=(1,2)) - np.sum(np.multiply(reg,kern==0), axis=(1,2))/np.count_nonzero(kern==0, axis=(1,2)))/255))

    return(slope_max, max_slope_angle,diff,max_diff_angle)

max_stars_repo_name: abidhaque/LineScore

max_issues_repo_name: abidhaque/LineScore

max_forks_repo_name: abidhaque/LineScore


text:
from backbone import Backbone
import torch
import argparse
import numpy as np
import time
import onnxruntime
import os

def to_numpy(tensor):
    return tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy()

def main(args):
    model = Backbone()
    if args.use_gpu:
        model.load_state_dict(torch.load(args.model_path), strict=True)
    else:
        model.load_state_dict(torch.load(args.model_path, map_location='cpu'), strict=True)

    device = 'cuda' if args.use_gpu else 'cpu'
    model = model.to(device)
    # print(model)

    model.eval()
    batch_size = 1
    x = torch.randn(batch_size, 3, args.input_size, args.input_size, requires_grad=True).to(device)
    torch_out = model(x)

    print("파이토치 모델 실행시간 측정")
    test_cnt = 5
    total_time = 0.
    for i in range(test_cnt):
        x = torch.randn(batch_size, 3, args.input_size, args.input_size, requires_grad=True).to(device)
        start = time.time()
        torch_out = model(x)
        total_time += time.time() - start
    pytorch_exec_time = total_time / test_cnt
    print("파이토치 모델 실행시간 측정 완료")

    os.makedirs(os.path.dirname(args.onnx_output_path), exist_ok=True)
    # 모델 변환
    torch.onnx.export(model,  # 실행될 모델
                      x,  # 모델 입력값 (튜플 또는 여러 입력값들도 가능)
                      args.onnx_output_path,  # 모델 저장 경로 (파일 또는 파일과 유사한 객체 모두 가능)
                      export_params=True,  # 모델 파일 안에 학습된 모델 가중치를 저장할지의 여부
                      # opset_version=11,          # 모델을 변환할 때 사용할 ONNX 버전
                      do_constant_folding=True,  # 최적하시 상수폴딩을 사용할지의 여부
                      input_names=['input'],  # 모델의 입력값을 가리키는 이름
                      output_names=['output'],  # 모델의 출력값을 가리키는 이름
                      # )
                      # dynamic_axes={'input': {2: 'height', 3: 'width'}}
                      )

    print("onnx 모델 컨버팅 완료")

    print("onnx 모델 로딩")
    ort_session = onnxruntime.InferenceSession(args.onnx_output_path)
    print("onnx 모델 로딩 완료")
    ort_inputs = {ort_session.get_inputs()[0].name: to_numpy(x)}
    ort_outs = ort_session.run(None, ort_inputs)

    print("onnx 모델 실행시간 측정")
    total_time = 0.
    for i in range(test_cnt):
        ort_inputs = {ort_session.get_inputs()[0].name: to_numpy(x)}

        start = time.time()
        ort_outs = ort_session.run(None, ort_inputs)
        total_time += time.time() - start
    onnx_exec_time = total_time / test_cnt
    print("파이토치 모델 실행시간 측정 완료")

    np.testing.assert_allclose(to_numpy(torch_out), ort_outs[0], rtol=1e-03, atol=1e-05)
    print("Exported model has been tested with ONNXRuntime, and the result looks good!")

    print("onnx 평균시간", onnx_exec_time, "파이토치 평균시간", pytorch_exec_time)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='for common image metric learning')

    parser.add_argument('--input_size', type=int, default=112)

    parser.add_argument('--model_path', default=None, type=str)
    parser.add_argument('--onnx_output_path', default=None, type=str)
    parser.add_argument('--use_gpu', default=False, action="store_true")

    main(parser.parse_args())


text:
# Lint as: python3
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Provides an implementation of object tracking using TF and TF-TRT."""

import copy
from typing import List
from typing import Union
import numpy as np
import tensorflow.compat.v1 as tf
from automl_video_ondevice.shot_classification.base_shot_classification import BaseShotClassificationInference
from automl_video_ondevice.shot_classification.config import ShotClassificationConfig
from automl_video_ondevice.types import ShotClassificationAnnotation
from automl_video_ondevice.types import Size

import automl_video_ondevice.utils as vot_utils
import tensorflow.contrib.tensorrt as trt  # pylint: disable=g-explicit-tensorflow-version-import,unused-import

class TFShotClassificationInference(BaseShotClassificationInference):
  """Implementation of the BaseShotClassificationInference using TF, or TF-TRT.

  TF-TRT is a hybrid solution, converting a sub-section of the Tensorflow graph
  into it's proprietary TensorRT format. This allows for maximum usage of
  NVIDIA devices while keeping full Tensorflow support.

  Non-TRT frozen graphs can also be used with this implementation.
  """

  def __init__(self, frozen_graph_path: str, label_map_path: str,
               config: ShotClassificationConfig):
    """Constructor for TFShotClassificationInference.

    Args:
      frozen_graph_path: String value for the file path of frozen graph.
      label_map_path: String value for the file path of the label map.
      config: ShotClassificationConfig object with shot classification configs.
    """
    self.config = config

    if self.config.top_k <= 0:
      raise ValueError(
          'Top k cannot be zero, or else no results will be returned.')

    self._load_label_map(label_map_path)
    self._load_frozen_graph(frozen_graph_path)

  def __del__(self):
    """Destructor for TFShotClassificationInference."""
    self.session.close()

  def _load_label_map(self, label_map_path: str):
    """Opens and parses a given shot classification label map into memory.

    The standard AutoML Video .pbtxt label map is assumed:
      item {name: "Cat" id: 0}
      item {name: "Dog" id: 1}

    Args:
      label_map_path: String value for the file path of the label map.
    """
    with open(label_map_path, 'r') as f:
      self.label_map, _ = vot_utils.parse_label_map(f.read())

  def _load_frozen_graph(self, frozen_graph_path: str):
    """Opens and loads a frozen graph into memory.

    Args:
      frozen_graph_path: String value for the file path of frozen graph.
    """
    frozen_graph = tf.GraphDef()
    with open(frozen_graph_path, 'rb') as f:
      frozen_graph.ParseFromString(f.read())

    self._is_lstm = self._check_lstm(frozen_graph)
    if self._is_lstm:
      print('Loading an LSTM model.')

    self.graph = tf.Graph()
    with self.graph.as_default():
      self.output_node = tf.import_graph_def(
          frozen_graph,
          return_elements=[
              'probabilities:0',
          ] + (['raw_outputs/lstm_c:0', 'raw_outputs/lstm_h:0']
               if self._is_lstm else []))
    self.session = tf.InteractiveSession(graph=self.graph)

    tf_probabilities = self.graph.get_tensor_by_name('import/probabilities:0')
    if self._is_lstm:
      tf_lstm_c = self.graph.get_tensor_by_name('import/raw_outputs/lstm_c:0')
      tf_lstm_h = self.graph.get_tensor_by_name('import/raw_outputs/lstm_h:0')

    self._output_nodes = [tf_probabilities
                         ] + ([tf_lstm_c, tf_lstm_h] if self._is_lstm else [])

    if self._is_lstm:
      self.lstm_c = np.ones(tf_lstm_c.get_shape())
      self.lstm_h = np.ones(tf_lstm_h.get_shape())

    self.sliding_window = None
    self.frames_since_last_inference = self.config.inference_rate
    self.last_annotations = []

  def _check_lstm(self, graph_def: tf.GraphDef) -> bool:
    """Checks to see if the input frozen graph is an LSTM graph.

    LSTM is determined by the existence of the raw_outputs/lstm_c and
    raw_outputs/lstm_h nodes. Some informational tensorflow node may be used in
    the future for determining the model type.

    Args:
      graph_def: Loaded frozen graph. This should not be a loaded tensorflow
        session, since LSTM needs to be determined before the model is loaded.
    Returns:
      If the frozen graph is of an LSTM model.
    """
    for node in graph_def.node:
      if node.name == 'raw_outputs/lstm_c' or node.name == 'raw_outputs/lstm_h':
        return True
    return False

  def input_size(self) -> Size:
    """Calculate / grab optimal input size.

    The user is expected to ensure the size of their input image is correct.
    This is in case the user wants to do any acceleration of image resizing
    themselves. Although this model accepts any input size, it is currently
    recommended that the input is the same as the size used in training.

    Returns:
      The expected input size, of the type Size.
    """
    return Size(256, 256)

  def run(self, timestamp: Union[int, float], frame: np.ndarray,
          annotations: List[ShotClassificationAnnotation]) -> bool:
    """Run inferencing for a single frame, to calculate annotations.

    Args:
      timestamp: Generally an integer representing the microsecond of the frame,
        however any unique number is also accepted.
      frame: A numpy array of the shape (h, w, 3), representing an RGB image.
        Each color channel should be a number [0, 256).
      annotations: A list to append the output annotations to. For normal use-
        case, this should be an empty list. The output annotations will be of
        type ShotClassificationAnnotation.

    Returns:
      A boolean, True if successful and False if unsuccessful.
    """
    np_frame = np.array(frame)

    # User has to handle frame resizing by themselves.
    if np.shape(np_frame)[0] != 256:
      raise ValueError(
          'Shot classification input must be a height of 256 pixels. '
          'There is no width limit. Aspect ratio must be retained.')

    if self.config.sliding_window_size == 1 or self._is_lstm:
      # A sliding window size 1 is just the current frame. It still needs to be
      # expanded for the input batch size.
      #
      # LSTM is forced to have a sliding window size 1.
      self.sliding_window = np.expand_dims(np_frame, axis=0)
    else:
      # Initiate sliding window if not created yet.
      if self.sliding_window is None:
        self.sliding_window = np.zeros(
            (self.config.sliding_window_size, np.shape(np_frame)[0],
             np.shape(np_frame)[1], 3))

      # Moves sliding window and adds new frame to the back
      # No need to move the sliding window if window size is only 1.
      self.sliding_window = np.roll(self.sliding_window, -1)
      self.sliding_window[self.config.sliding_window_size - 1] = np_frame

    self.frames_since_last_inference += 1
    if self.frames_since_last_inference >= self.config.inference_rate or self._is_lstm:
      self.frames_since_last_inference = 0
      with self.graph.as_default():
        # Tensors to feed in.
        feed_dict = {
            'import/video_inputs:0': self.sliding_window,
        }
        if self._is_lstm:
          feed_dict.update({
              'import/raw_inputs/init_lstm_c:0': self.lstm_c,
              'import/raw_inputs/init_lstm_h:0': self.lstm_h
          })

        session_return = self.session.run(
            self._output_nodes, feed_dict=feed_dict)
        if self._is_lstm:
          (probabilities, self.lstm_c, self.lstm_h) = session_return
          score = probabilities
        else:
          (probabilities) = session_return
          score = probabilities[0]

        assert len(self.label_map) == len(score)

        for i in range(len(score)):
          if score[i] < self.config.score_threshold:
            continue

          annotation = ShotClassificationAnnotation(
              timestamp=timestamp,
              class_name=self.label_map[i],
              confidence_score=score[i],
          )
          annotations.append(annotation)
        annotations.sort(key=lambda v: v.confidence_score, reverse=True)

        # Removes everything not in the top k.
        if self.config.top_k > 0:
          del annotations[self.config.top_k:]

        if self.config.duplicate_results:
          self.last_annotations = copy.deepcopy(annotations)
    else:
      if self.config.duplicate_results:
        annotations.extend(copy.deepcopy(self.last_annotations))
        for annotation in annotations:
          annotation.timestamp = timestamp
      else:
        annotations.append(None)

    return True

max_stars_repo_name: sujitahirrao/automl-video-ondevice

max_issues_repo_name: sujitahirrao/automl-video-ondevice

max_forks_repo_name: sujitahirrao/automl-video-ondevice


text:
import cv2
import numpy as np
from tqdm.auto import tqdm

from . import MarkupEngine
from .._base import Visualiser_Base

class Visualiser(Visualiser_Base):
    def __init__(self, video_io, model, cmap=dict()):
        """Create a new visualiser object.

        Parameters
        ----------
        video_io : str or cv2.VideoCapture
            If string, will create a new cv2.VideoCapture object from filename.
        model : src.Model._base.ModelBase
            A model class. Must implement the method `predict(self, frame, prevFrame)
        cmap : dict
            Dictionary mapping labels to particular labels. Keys should be labels, values should be colours. Possible values are: navy, blue, aqua, teal, olive, green, lime, yellow, orange, red, maroon, fuchsia, purple, black, gray , silver. If None is given, a random colour will be chosen.
        """
        super().__init__(video_io)
        self.MUE = MarkupEngine(model)
        self.cmap = cmap

    def next_markup(self):
        """
        Get the next marked up frame
        """
        hasFrame, frame = self.next_frame()
        if hasFrame:
            curFrameNo = self._get_frameNum()
            frame = self.MUE.markup(frame, curFrameNo, self.prevFrame, cmap=self.cmap)
        return hasFrame, frame

    def markup_all(self, fname, codec="MJPG", verbose=True, show=False):
        if not show:
            vw = self._create_new_writer(fname, codec)
            print("vw created")
        else:
            window = cv2.namedWindow(fname)
        self.reset_seed()

        # Range of frames to iterate over
        fCount = self._get_fCount(verbose)

        # Iterate through each frame
        for fi in fCount:
            hasFrame, frame = self.next_markup()

            # Break early if we have no frame
            if not hasFrame:
                break

            if show:
                frame = self._cvt_BGR2RGB(frame)
                # Show the frame
                cv2.imshow(fname, frame)
                # Press Q on keyboard to  exit
                if cv2.waitKey(self.msPerFrame) & 0xFF == ord("q"):
                    break
            else:
                # Write to the video writer
                self._cv_write_frame(vw, frame)

        # Reset the position
        self.reset_seed()
        if not show:
            vw.release()

max_stars_repo_name: celsomilne/soccer

max_issues_repo_name: celsomilne/soccer

max_forks_repo_name: celsomilne/soccer


text:
import sys
import os
from pathlib import Path
import time
import numpy as np
import random

path = Path(os.path.abspath(__file__))
# This corresponds to DeepHyperion-BNG
sys.path.append(str(path.parent))
sys.path.append(str(path.parent.parent))

from core.folder_storage import SeedStorage
import glob, json
from random import shuffle, choice
from self_driving.road_bbox import RoadBoundingBox
from self_driving.beamng_individual import BeamNGIndividual
import core.utils as us
from scipy.spatial import distance
from core.feature_dimension import FeatureDimension
from self_driving.simulation_data import SimulationParams, SimulationData, SimulationDataRecord, SimulationInfo
from self_driving.vehicle_state_reader import VehicleState
from self_driving.beamng_member import BeamNGMember
from self_driving.decal_road import DecalRoad

def get_spine(member):
    print("member: ", member)
    with open(member) as json_file:
        spine = json.load(json_file)
        return spine['sample_nodes']

def get_min_distance_from_set(ind, solution):
    distances = list()
    ind_spine = ind[0]

    for road in solution:
        road_spine = road[0]
        distances.append(manhattan_dist(ind_spine, road_spine))
    distances.sort()
    return distances[0]

def manhattan_dist(ind1, ind2):
    return distance.cityblock(list(ind1), list(ind2))

def initial_pool_generator(config, problem):
    good_members_found = 0
    attempts = 0
    storage = SeedStorage('initial_pool')
    i = 0
    while good_members_found < config.POOLSIZE:#40:
        path = storage.get_path_by_index(good_members_found + 1)
        attempts += 1
        print(f'attempts {attempts} good {good_members_found} looking for {path}')

        # rand = random.uniform(0,1)
        # if rand < 0.3:
        #     max_angle = 80
        # else:
        #     max_angle = random.randint(10,100)
        max_angle = 80

        member = problem.generate_random_member(max_angle)
        member.evaluate()
        if member.distance_to_boundary <= 0:
            continue
        member.config = config
        member.problem = problem
        good_members_found += 1
        path.write_text(json.dumps({
            "control_nodes": member.control_nodes,
            member.simulation.f_params: member.simulation.params._asdict(),
            member.simulation.f_info: member.simulation.info.__dict__,
            member.simulation.f_road: member.simulation.road.to_dict(),
            member.simulation.f_records: [r._asdict() for r in member.simulation.states]
        }))
    return storage.folder

def initial_population_generator(path, config, problem):
    all_roads = [filename for filename in glob.glob(str(path)+"\*.json", recursive=True)]
    type = config.Feature_Combination
    shuffle(all_roads)

    roads = all_roads[:40]

    original_set = list()

    individuals = []
    popsize = config.POPSIZE
    i = 0
    for road in roads:
        with open(road) as json_file:
            data = json.load(json_file)
            sample_nodes = data["road"]["nodes"]
            for node in sample_nodes:
                node[2] = -28.0
            sample_nodes = np.array(sample_nodes)
            records = data["records"]

        bbox_size = (-250.0, 0.0, 250.0, 500.0)
        road_bbox = RoadBoundingBox(bbox_size)
        member = BeamNGMember(data["control_nodes"], [tuple(t) for t in sample_nodes], len(data["control_nodes"]),
                              road_bbox)
        member.config = config
        member.problem = problem
        simulation_id = time.strftime('%Y-%m-%d--%H-%M-%S', time.localtime())
        sim_name = member.config.simulation_name.replace('$(id)', simulation_id)
        simulation_data = SimulationData(sim_name)
        states = []
        for record in records:
            state = VehicleState(timer=record["timer"]
                                 , pos=record["pos"]
                                 , dir=record["dir"]
                                 , vel=record["vel"]
                                 , steering=record["steering"]
                                 , steering_input=record["steering_input"]
                                 , brake=record["brake"]
                                 , brake_input=record["brake_input"]
                                 , throttle=record["throttle"]
                                 , throttle_input=record["throttle_input"]
                                 , wheelspeed=record["wheelspeed"]
                                 , vel_kmh=record["vel_kmh"])

            sim_data_record = SimulationDataRecord(**state._asdict(),
                                                   is_oob=record["is_oob"],
                                                   oob_counter=record["oob_counter"],
                                                   max_oob_percentage=record["max_oob_percentage"],
                                                   oob_distance=record["oob_distance"])
            states.append(sim_data_record)

        simulation_data.params = SimulationParams(beamng_steps=data["params"]["beamng_steps"],
                                                  delay_msec=int(data["params"]["delay_msec"]))

        simulation_data.road = DecalRoad.from_dict(data["road"])
        simulation_data.info = SimulationInfo()
        simulation_data.info.start_time = data["info"]["start_time"]
        simulation_data.info.end_time = data["info"]["end_time"]
        simulation_data.info.success = data["info"]["success"]
        simulation_data.info.computer_name = data["info"]["computer_name"]
        simulation_data.info.id = data["info"]["id"]

        simulation_data.states = states
        if len(states) > 0:
            member.distance_to_boundary = simulation_data.min_oob_distance()
            member.simulation = simulation_data
            individual: BeamNGIndividual = BeamNGIndividual(member, config)
        #individual.evaluate()
        b = tuple()
        feature_dimensions = generate_feature_dimension(type)
        for ft in feature_dimensions:
            i = feature_simulator(ft.feature_simulator, individual)
            b = b + (i,)
        individuals.append([b, road, individual])

    starting_point = choice(individuals)
    original_set.append(starting_point)

    i = 0
    best_ind = individuals[0]
    while i < popsize - 1:
        max_dist = 0
        for ind in individuals:
            dist = get_min_distance_from_set(ind, original_set)
            if dist > max_dist:
                max_dist = dist
                best_ind = ind
        original_set.append(best_ind)
        i += 1

    base = config.initial_population_folder
    storage = SeedStorage(base)
    for index, road in enumerate(original_set):
        dst = storage.get_path_by_index(index + 1)
        ind = road[2]
        #copy(road[1], dst)
        with open(road[1]) as ff:
            json_file = json.load(ff)
        with open(dst, 'w') as f:
            f.write(json.dumps({
                "control_nodes": json_file["control_nodes"],
                ind.m.simulation.f_params: ind.m.simulation.params._asdict(),
                ind.m.simulation.f_info: ind.m.simulation.info.__dict__,
                ind.m.simulation.f_road: ind.m.simulation.road.to_dict(),
                ind.m.simulation.f_records: [r._asdict() for r in ind.m.simulation.states]
            }))

def feature_simulator(function, x):
    """
    Calculates the number of control points of x's svg path/number of bitmaps above threshold
    :param x: genotype of candidate solution x
    :return:
    """
    if function == 'min_radius':
        return us.min_radius(x)
    if function == 'mean_lateral_position':
        return us.mean_lateral_position(x)
    if function == "dir_coverage":
        return us.direction_coverage(x)
    if function == "segment_count":
        return us.segment_count(x)
    if function == "sd_steering":
        return us.sd_steering(x)
    if function == "curvature":
        return us.curvature(x)

def generate_feature_dimension(_type):
    fts = list()

    if _type == 0:
        ft1 = FeatureDimension(name="MinRadius", feature_simulator="min_radius", bins=1)
        fts.append(ft1)
        ft2 = FeatureDimension(name="DirectionCoverage", feature_simulator="dir_coverage", bins=1)
        fts.append(ft2)

    elif _type == 1:
        ft1 = FeatureDimension(name="MinRadius", feature_simulator="min_radius", bins=1)
        fts.append(ft1)

        ft3 = FeatureDimension(name="MeanLateralPosition", feature_simulator="mean_lateral_position", bins=1)
        fts.append(ft3)

    elif _type == 2:
        ft2 = FeatureDimension(name="DirectionCoverage", feature_simulator="dir_coverage", bins=1)
        fts.append(ft2)

        ft3 = FeatureDimension(name="MeanLateralPosition", feature_simulator="mean_lateral_position", bins=1)
        fts.append(ft3)

    elif _type == 3:
        ft1 = FeatureDimension(name="MinRadius", feature_simulator="min_radius", bins=1)
        fts.append(ft1)
        ft2 = FeatureDimension(name="SegmentCount", feature_simulator="segment_count", bins=1)
        fts.append(ft2)

    elif _type == 4:
        ft1 = FeatureDimension(name="SegmentCount", feature_simulator="segment_count", bins=1)
        fts.append(ft1)

        ft3 = FeatureDimension(name="MeanLateralPosition", feature_simulator="mean_lateral_position", bins=1)
        fts.append(ft3)

    elif _type == 5:
        ft2 = FeatureDimension(name="DirectionCoverage", feature_simulator="dir_coverage", bins=1)
        fts.append(ft2)

        ft3 = FeatureDimension(name="SegmentCount", feature_simulator="segment_count", bins=1)
        fts.append(ft3)

    elif _type == 6:
        ft2 = FeatureDimension(name="SDSteeringAngle", feature_simulator="sd_steering", bins=1)
        fts.append(ft2)

        ft3 = FeatureDimension(name="SegmentCount", feature_simulator="segment_count", bins=1)
        fts.append(ft3)

    elif _type == 7:
        ft2 = FeatureDimension(name="SDSteeringAngle", feature_simulator="sd_steering", bins=1)
        fts.append(ft2)

        ft3 = FeatureDimension(name="DirectionCoverage", feature_simulator="dir_coverage", bins=1)
        fts.append(ft3)

    elif _type == 8:
        ft2 = FeatureDimension(name="SDSteeringAngle", feature_simulator="sd_steering", bins=1)
        fts.append(ft2)

        ft3 = FeatureDimension(name="MinRadius", feature_simulator="min_radius", bins=1)
        fts.append(ft3)

    elif _type == 9:
        ft2 = FeatureDimension(name="SDSteeringAngle", feature_simulator="sd_steering", bins=1)
        fts.append(ft2)

        ft3 = FeatureDimension(name="MeanLateralPosition", feature_simulator="mean_lateral_position", bins=1)
        fts.append(ft3)

    elif _type == 10:
        ft2 = FeatureDimension(name="SDSteeringAngle", feature_simulator="sd_steering", bins=1)
        fts.append(ft2)

        ft3 = FeatureDimension(name="Curvature", feature_simulator="curvature", bins=1)
        fts.append(ft3)

    elif _type == 11:
        ft2 = FeatureDimension(name="SegmentCount", feature_simulator="segment_count", bins=1)
        fts.append(ft2)

        ft3 = FeatureDimension(name="Curvature", feature_simulator="curvature", bins=1)
        fts.append(ft3)

    elif _type == 12:
        ft2 = FeatureDimension(name="MeanLateralPosition", feature_simulator="mean_lateral_position", bins=1)
        fts.append(ft2)

        ft3 = FeatureDimension(name="Curvature", feature_simulator="curvature", bins=1)
        fts.append(ft3)

    return fts

if __name__ == '__main__':
    path = initial_pool_generator()
    #path = r"C:\Users\Aurora\new-DeepJanus\DeepJanus\DeepJanus-BNG\data\member_seeds\initial_pool"
    initial_population_generator(path)

max_stars_repo_name: IharBakhanovich/DeepHyperion

max_issues_repo_name: IharBakhanovich/DeepHyperion

max_forks_repo_name: IharBakhanovich/DeepHyperion