text:
# -*- coding: utf-8 -*-

from recurrent_controller import RecurrentController
import sys
sys.path.append('./dnc')
from dnc.dnc import DNC
import tensorflow as tf
import numpy as np
import pickle
import json
import sys
import os
import re
import csv
import statistics

def llprint(message):
    sys.stdout.write(message)
    sys.stdout.flush()

def load(path):
    return pickle.load(open(path, 'rb'))

def onehot(index, size):
    vec = np.zeros(size, dtype=np.float32)
    vec[index] = 1.0
    return vec

def prepare_sample(sample, target_code, word_space_size, lexicon_dictionary):
    # input_vec = sample[:sample.index(target_code)]
    # output_vec = np.array(sample, dtype=np.float32)
    # while len(input_vec) < len(output_vec):
    #     input_vec.append(target_code)
    # input_vec = np.array(input_vec, dtype=np.float32)
    input_vec = np.array(sample[:sample.index(target_code)], dtype=np.float32)
    output_vec = sample[sample.index(target_code) + 1:]
    while len(output_vec) < len(input_vec):
        output_vec.append(target_code)
    output_vec = np.array(output_vec, dtype=np.float32)
    seq_len = input_vec.shape[0]
    weights_vec = np.zeros(seq_len, dtype=np.float32)

    # target_mask = (input_vec == target_code)
    target_mask = (output_vec != target_code)
    weights_vec[target_mask] = 1.0
    #print("Input vector: ", [list(lexicon_dictionary.keys())[int(num)] for num in input_vec])
    #print("Output vector: ", [list(lexicon_dictionary.keys())[int(num)] for num in output_vec])
    input_vec = np.array([onehot(int(code), word_space_size) for code in input_vec])
    output_vec = np.array([onehot(int(code), word_space_size) for code in output_vec])

    return (
        np.reshape(input_vec, (1, -1, word_space_size)),
        np.reshape(output_vec, (1, -1, word_space_size)),
        seq_len,
        np.reshape(weights_vec, (1, -1, 1))
    )

if __name__ == '__main__':
    ckpts_dir = './checkpoints/word_count_256/'
    lexicon_dictionary = load('./data/encoded/lexicon-dict.pkl')
    question_code = lexicon_dictionary["="]
    #target_code = lexicon_dictionary["-"]
    test_files = []

    for entry_name in os.listdir('./data/encoded/test/'):
        entry_path = os.path.join('./data/encoded/test/', entry_name)
        if os.path.isfile(entry_path):
            test_files.append(entry_path)

    for filename in [os.path.join('data', 'annotator1_test.txt'), os.path.join('data', 'annotator2_test.txt')]:
        with open(filename) as csvfile:
            readCSV = csv.reader(csvfile, delimiter='\t')
            question_types = {}
            for row in readCSV:
                task_number = row[0]
                question_type = row[2]
                if task_number in question_types:
                    if len(question_types[task_number]) < 2:
                        question_types[task_number].append(question_type)
                else:
                    question_types[task_number] = [question_type]

    graph = tf.Graph()
    with graph.as_default():
        with tf.compat.v1.Session(graph=graph) as session:

            ncomputer = DNC(
                RecurrentController,
                input_size=len(lexicon_dictionary),
                output_size=len(lexicon_dictionary),
                max_sequence_length=100,
                memory_words_num=256,
                memory_word_size=64,
                memory_read_heads=4,
            )

            checkpoints = os.listdir(ckpts_dir)
            if len(checkpoints) != 0:
                checkpoint_numbers = [int(checkpoint[checkpoint.find("-") + 1:]) for checkpoint in checkpoints if
                                      checkpoint[checkpoint.find("-") + 1:].isnumeric()]
                checkpoint_numbers.sort()
                ncomputer.restore(session, ckpts_dir, f"step-{checkpoint_numbers[-1]}")
            else:
                raise FileNotFoundError("No checkpoint to test.")

            outputs, _ = ncomputer.get_outputs()
            softmaxed = tf.nn.softmax(outputs)

            test_names = []
            test_data = []
            test_data_types = []
            for counter, test_file in enumerate(test_files):
                task_regexp = r'([0-9])+-([0-9])+_test([0-9])+.json'
                task_filename = os.path.basename(test_file)
                task_match_obj = re.match(task_regexp, task_filename)
                if task_match_obj:
                    with open(test_file) as f:
                        test_data.append(json.load(f))
                    task_number = task_match_obj.group(0)[:task_match_obj.group(0).rfind('_')]
                    if task_number in question_types:
                        task_types = question_types[task_number]
                    else:
                        task_types = None
                    test_data_types.append(task_types)
                    task_name = f"Test {test_file[test_file.find(os.path.basename(test_file)):test_file.rfind('_')]} (Question {test_file[-6]})"
                    test_names.append(task_name)
                    llprint("\r%s ... %d/%d" % (task_name, counter + 1, len(test_files)))

            results = []
            tasks_results = {}
            questions_and_answers = open("test_responses_answer_only.csv", "w+")
            questions_and_answers.write(f"Task Name\tDNC's Answer\tExpected Answer\tGrade\n")
            for i, story in enumerate(test_data):
                question_index = story.index(question_code)

                #desired_answers = np.array(story)
                input_vec, desired_answers, seq_len, _ = prepare_sample(story, lexicon_dictionary['='], len(lexicon_dictionary), lexicon_dictionary)
                desired_answers = [np.where(one_hot_arr == 1)[0][0] for one_hot_arr in desired_answers[0]]
                softmax_output = session.run(softmaxed, feed_dict={
                    ncomputer.input_data: input_vec,
                    ncomputer.sequence_length: seq_len
                })

                softmax_output = np.squeeze(softmax_output, axis=0)
                given_answers = np.argmax(softmax_output, axis=1)

                answers_cursor = 0
                question_grade = []
                targets_cursor = question_index + 1
                while targets_cursor < len(story):
                    question_grade.append(given_answers[answers_cursor] == desired_answers[answers_cursor])
                    answers_cursor += 1
                    targets_cursor += 1
                results.append(np.prod(question_grade))
                if test_data_types[i] is not None:
                    for annotator_str in test_data_types[i]:
                        for char in annotator_str:
                            if char in tasks_results:
                                tasks_results[char].append(np.prod(question_grade))
                            else:
                                tasks_results[char] = [np.prod(question_grade)]

                llprint("\r%s ... %d/%d" % (test_names[i], i, len(test_data)))
                # print("Given Answer: ", given_answers)
                # print("Desired Answer: ", desired_answers)
                # print("Question grade: ", question_grade)
                word_given_answer = [list(lexicon_dictionary.keys())[num] for num in given_answers]
                print(word_given_answer)
                word_desired_answer = [list(lexicon_dictionary.keys())[num] for num in desired_answers]
                questions_and_answers.write(f"{test_names[i]}\t{word_given_answer}\t{word_desired_answer}\t{question_grade}\n")
            questions_and_answers.close()
            error_rate = 1. - np.mean(results)
            llprint("\r%s ... %.3f%% Error Rate.\n" % (task_name, error_rate * 100))

            print("\n")
            print("%-27s%-27s%s" % ("Task", "Mean % Error", "Standard Deviation"))
            print("-------------------------------------------------------------------")
            means = []
            for task in tasks_results.keys():
                means.append(np.mean(tasks_results[task]))
                print("%-27s%-27s%s" % (task, (1 - means[-1]) * 100, statistics.stdev([(1 - result) * 100 for result in tasks_results[task]])))
            print("-------------------------------------------------------------------")
            results_mean = "%.2f%%" % (1 - np.mean(results) * 100)
            failed_count = 0
            for mean in means:
                if mean < .95:
                    failed_count += 1
            failed_count = "%d" % (failed_count)

            print("%-27s%-27s" % ("Mean Err.", results_mean))
            failed_count = 0
            for mean in means:
                if mean < .95:
                    failed_count += 1
            print("%-27s%-27s" % ("Failed (err. > 5%)", failed_count))

max_stars_repo_name: egoetz/DNC-tensorflow

max_issues_repo_name: egoetz/DNC-tensorflow

max_forks_repo_name: egoetz/DNC-tensorflow


text:
"""
Here we provide codes to compute c.n.i.d, Mo and Or mentioned in the article.
Some codes are inspired by 'Hadian, Raheleh, Blazej Grabowski, and Jörg Neugebauer.
 "GB code: A grain boundary generation code." The Journal of Open Access Software 3 (2018)',
and Banadaki, Arash D., and Srikanth Patala. "An efficient algorithm for computing the primitive
bases of a general lattice plane." Journal of Applied Crystallography 48.2 (2015): 585-588.
"""

import numpy as np
from numpy import dot, cross, square
from numpy.linalg import norm, inv

def get_fraction_basis(B):
    """
    out put an string of a basis with rational coefficients
    """
    B_c = np.empty_like(B)
    B_c[:, 0], n1 = find_integer_vectors(B[:, 0], 10000)
    B_c[:, 1], n2 = find_integer_vectors(B[:, 1], 10000)
    B_c = np.array(B_c, dtype=int)
    B_c = np.array(B_c, dtype=str)
    for i in range(3):
        B_c[:, 0][i] = B_c[:, 0][i] + "/" + str(n1)
        B_c[:, 1][i] = B_c[:, 1][i] + "/" + str(n2)
    return B_c

def get_coef_exp_ltc(lattice, carte):
    """
    convert a set of column vectors expressed in the cartesian coordinates
    to be expressed in a lattice coordinates
    arguments:
    lattice -- lattice basis
    carte -- column vectors cartesian coordinates
    """
    return dot(inv(lattice), carte)

def ang(a, b):
    """
    Returns value of cos(theta), where theta is the angle between vectors: a and b
    """
    with np.errstate(divide='ignore', invalid='ignore'):
        return dot(a, b) / (norm(a) * norm(b))

def rot(axis, theta):
    """Returns rotation matrix around axis with angle of theta in radian"""
    axis = np.array(axis) / norm(axis)
    return (
        np.cos(theta) * np.eye(3)
        + np.sin(theta) * np.array([
            [0., -axis[2], axis[1]],
            [axis[2], 0., -axis[0]],
            [-axis[1], axis[0], 0.]
        ])
        + (1.-np.cos(theta)) * np.outer(axis, axis)
    )

def signed_cell_volume(cell):
    """Returns signed cell volume by scalar triple product"""
    return dot(cell[:, 0], cross(cell[:, 1], cell[:, 2]))

def plane_cell_area(cell):
    """Returns signed cell volume by scalar triple product"""
    return norm(cross(cell[:, 1], cell[:, 2]))

def reciprocal_cell(cell):
    """Returns reciprocal cell (without factor of 2PI)"""
    v = signed_cell_volume(cell)
    recipro_cell = np.empty((3, 3))
    recipro_cell[:, 0] = cross(cell[:, 1], cell[:, 2]) / v
    recipro_cell[:, 1] = cross(cell[:, 2], cell[:, 0]) / v
    recipro_cell[:, 2] = cross(cell[:, 0], cell[:, 1]) / v
    return recipro_cell

def Basis(basis):
    """
    basis vectors of fcc and bcc lattice
    """
    if basis == 'fcc':
        return np.array([[0.5, 0, 0.5],
                         [0.5, 0.5, 0],
                         [0, 0.5, 0.5]], dtype=np.float64)
    elif basis == 'bcc':
        return np.array([[0.5, 0.5, 0.5],
                        [-0.5, -0.5, 0.5],
                        [0.5, -0.5, -0.5]], dtype=np.float64)
    else:
        ValueError('only available for fcc and bcc lattices to search the primitive CSL cell')

def find_integer_vectors(v, N_max, tol=1e-6):
    # A function find the natural number N up to N_max so that Nv contains only
    # integer elements with tolerance of tol and gcd(v1,v2,v3,N) = 1
    for N in range(1, N_max + 1):
        Nv = v * N
        if np.all(abs(Nv - np.round(Nv)) < tol):
            break
    else:
        raise RuntimeError('failed to find the rational vector of ' + str(v) + '\n within denominator <= ' + str(N_max))
    return np.round(Nv).astype(int), N

def projection(u1, u2):
    # get the projection of u1 on u2
    return dot(u1, u2) / dot(u2, u2)

def Gram_Schmidt(B0):
    # Gram–Schmidt process
    Bstar = np.eye(3, len(B0.T))
    for i in range(len(B0.T)):
        if i == 0:
            Bstar[:, i] = B0[:, i]
        else:
            BHere = B0[:, i].copy()
            for j in range(i):
                BHere = BHere - projection(B0[:, i], B0[:, j]) * B0[:, j]
            Bstar[:, i] = BHere
    return Bstar

def LLL(B):
    # LLL lattice reduction algorithm
    # https://en.wikipedia.org/wiki/Lenstra%E2%80%93Le-
    # nstra%E2%80%93Lov%C3%A1sz_lattice_basis_reduction_algorithm
    Bhere = B.copy()
    Bstar = Gram_Schmidt(Bhere)
    delta = 3 / 4
    k = 1
    while k <= len(B.T) - 1:
        js = -np.sort(-np.arange(0, k + 1 - 1))
        for j in js:
            ukj = projection(Bhere[:, k], Bstar[:, j])
            if abs(ukj) > 1 / 2:
                Bhere[:, k] = Bhere[:, k] - round(ukj) * Bhere[:, j]
                Bstar = Gram_Schmidt(Bhere)
        ukk_1 = projection(Bhere[:, k], Bstar[:, k - 1])
        if dot(Bstar[:, k], Bstar[:, k]) >= (delta - square(ukk_1)) * dot(Bstar[:, k - 1],Bstar[:, k - 1]):
            k += 1
        else:
            m = Bhere[:, k].copy()
            Bhere[:, k] = Bhere[:, k - 1].copy()
            Bhere[:, k - 1] = m
            Bstar = Gram_Schmidt(Bhere)
            k = max(k - 1, 1)
    return Bhere

def MID(lattice, n, tol = 1e-10, N_max=10000):
    # get the miller indices of a lattice plane with a normal vector n
    for i in range(3):
        if abs(dot(lattice[:, i], n)) > tol:
            Pc1 = lattice[:, i]
            break
    hkl = get_indices_from_n_Pc1(n, lattice, Pc1)
    hkl = find_integer_vectors(hkl, N_max)[0]
    return hkl

def get_primitive_hkl(hkl, C_lattice, P_lattice, N_max=10000):
    # convert the miller indices from conventional cell to primitive cell
    # 1. get normal
    n, Pc1 = get_plane(hkl, C_lattice)
    hkl_p = get_indices_from_n_Pc1(n, P_lattice, Pc1)
    hkl_p = find_integer_vectors(hkl_p, N_max)[0]
    return hkl_p

def get_plane(hkl, lattice):
    # get the normal vector and one in-plane point for the (hkl) plane of the lattice
    points = np.eye(3)
    for i in range(3):
        if hkl[i] != 0:
            points[:, 0] = lattice[:, i] / hkl[i]
            count = i
            break
    count2 = 1
    for i in range(3):
        if i != count:
            if hkl[i] == 0:
                points[:, count2] = points[:, 0] + lattice[:, i]
            else:
                points[:, count2] = lattice[:, i] / hkl[i]
            count2 += 1
    n = cross((points[:, 0] - points[:, 1]), (points[:, 0] - points[:, 2]))
    return n, points[:, 0]

def get_indices_from_n_Pc1(n, lattice, Pc1):
    # get the miller indices of certain plane with normal n
    # and one in-plane point Pc1 for certain lattice
    hkl = np.empty(3)
    for i in range(3):
        hkl[i] = dot(lattice[:, i], n) / dot(Pc1, n)
    return hkl

def ext_euclid(a, b):
    # extended euclidean algorithm
    # from https://en.wikipedia.org/wiki/Extended_Euclidean_algorithm
    old_s, s = 1, 0
    old_t, t = 0, 1
    old_r, r = a, b
    if b == 0:
        return 1, 0, a
    else:
        while(r != 0):
            q = old_r // r
            old_r, r = r, old_r - q * r
            old_s, s = s, old_s - q * s
            old_t, t = t, old_t - q * t
    return old_s, old_t, old_r

def get_pri_vec_inplane(hkl, lattice, N_max=1000):
    # get two primitive lattice vector
    h, k, l = hkl
    if k == 0 and l == 0:
        return LLL(np.column_stack((lattice[:, 1], lattice[:, 2])))
    by, bz, c = ext_euclid(abs(k), abs(l))
    if h == 0:
        v1 = np.array([1, 0, 0])
        v2 = np.array([0, -l/c, k/c])
    else:
        bx = -c / h
        if k != 0:
            by = k / abs(k) * by
        if l != 0:
            bz = l / abs(l) * bz
        v1 = np.array([0, -l / c, k / c])
        v2 = np.array([bx, by, bz])
    v2 = find_integer_vectors(v2, N_max)[0]
    return LLL(dot(lattice, np.column_stack((v1, v2))))

def get_two_bases(lattice_1, lattice_2, R, miller_indices):
    """
    get the two plane bases in a coincident plane of lattice 1 & lattice 2
    arguments:
    lattice_1 --- column vectors of lattice 1
    lattice_2
    R --- rotation matrix applied on lattice 2
    miller_indices --- expressing the GB plane in lattice 1 frame
    return:
    B1, B2 --- two plane bases
    """
    PB_1 = get_pri_vec_inplane(miller_indices, lattice_1)
    lattice_2 = dot(R, lattice_2)
    normal = cross(PB_1[:, 0], PB_1[:, 1])
    miller_indices_2 = MID(lattice_2, normal)
    PB_2 = get_pri_vec_inplane(miller_indices_2, lattice_2)
    return PB_1, PB_2

def search_CSL(B1, B2, lim, ortho = False, tol = 1e-4):
    """
    finds CSL of two 2-D lattices
    arguments:
    B1 -- 3D basis
    B2 -- plane basis
    lim -- control num of generated reciprocal lattice points to find CSL
    ortho -- whether to obtain two orthogonal vectors
    tol -- tolerance to judge whether the two vectors are orthogonal
    return:
    v1, v2 -- two basic vectors of CSL
    """
    B1 = np.column_stack((cross(B1[:, 0], B1[:, 1]), B1))
    basis1 = B1.T
    basis2 = B2.T

    # meshes
    x = np.arange(-lim, lim + 1, 1)
    y = x

    indice = (np.stack(np.meshgrid(x, y)).T).reshape(len(x) ** 2, 2)
    indice_0 = indice[np.where(np.sum(abs(indice), axis=1) != 0)]
    indice_0 = np.array(indice_0, dtype=np.float64)
    # Reciprocal lattice points in the GB plane
    LP2 = dot(indice_0, basis2)

    # describe lattice-2 in coordinate-1
    L2f = dot(LP2, inv(basis1))
    # get CSL points
    L2f = L2f[np.where(np.all(abs(np.round(L2f) - L2f) < 1e-7, axis=1))[0]]

    # convert to cartesian coordinate
    L2f = dot(L2f, basis1)
    L2f = L2f[np.argsort(norm(L2f, axis=1))]

    # get the two minimum non linear vectors
    v1 = L2f[0]
    for i in L2f:
        if ortho == False:
            if (1 - abs(ang(i, v1)) > 1e-7):
                v2 = i
                found = True
                break
        else:
            if abs(dot(i, v1)) < tol:
                v2 = i
                found = True
                break

    else:
        raise RuntimeError('failed to find the csl of the reciprocal lattices in the gb plane, \
        maybe the inputted two lattices do not form a CSL, or please increase lim')

    return LLL(np.column_stack((v1, v2)))

def searchcnid(B1, B2, lim):
    """
    finds c.n.i.d
    For more details of c.n.i.d,
    refer the book: 'Interfaces in crystalline materials',
     Sutton and Balluffi, clarendon press, 1996.
    arguments:
    B1 -- plane basis 1
    B2 -- plane basis 2
    lim -- control num of generated reciprocal lattice points to find CSL
    return:
    B -- two column vectors of CNID
    """
    # normal vector
    v_n = cross(B1[:, 0], B1[:, 1]) / norm(B1[:, 0])
    RP1 = np.column_stack((v_n, B1))
    RP2 = np.column_stack((v_n, B2))

    # Compute the reciprocal Lattices
    R1 = reciprocal_cell(RP1)
    R2 = reciprocal_cell(RP2)

    # get the CSL of the 2D reciprocal lattice
    basis1 = np.column_stack((R1[:, 1], R1[:, 2]))
    basis2 = np.column_stack((R2[:, 1], R2[:, 2]))

    v1, v2 = search_CSL(basis1, basis2, lim).T

    cnidP = np.empty((3, 3))
    cnidP[:, 0] = R1[:, 0]
    cnidP[:, 1] = v1
    cnidP[:, 2] = v2
    # convert to direct lattice
    cnid = reciprocal_cell(cnidP)
    if norm(cnid[:, 1]) > norm(cnid[:, 2]):
        copyv = cnid[:, 1].copy()
        cnid[:, 1] = cnid[:, 2]
        cnid[:, 2] = copyv
    if signed_cell_volume(cnid) < 0:
        cnid[:, 0] = -cnid[:, 0]

    return LLL(np.column_stack((cnid[:, 1], cnid[:, 2])))

ext: py

max_stars_repo_name: nmdl-mizo/cnidcal

max_issues_repo_name: nmdl-mizo/cnidcal

max_forks_repo_name: nmdl-mizo/cnidcal

reason: import numpy,from numpy


text:
import warnings
warnings.filterwarnings("ignore")
import numpy as np
import matplotlib.pyplot as plt
from set_style import set_style

set_style('default', w=1, h=3.5)

fig, [ax1, ax2, ax3, ax4, ax5] = plt.subplots(5,1)

data = np.load('../data/figure4.npz')

t = data['t']
phi_msn = data['phi_msn']
phi_mdn = data['phi_mdn']
phi_msg = data['phi_msg']
phi_mdg = data['phi_mdg']

Na_se = data['cNa_se']
K_se = data['cK_se']
Cl_se = data['cCl_se']
Ca_se = data['cCa_se']
Na_de = data['cNa_de']
K_de = data['cK_de']
Cl_de = data['cCl_de']
Ca_de = data['cCa_de']

V_sn = data['V_sn']
V_se = data['V_se']
V_sg = data['V_sg']
V_dn = data['V_dn']
V_de = data['V_de']
V_dg = data['V_dg']

### Panel A ###
l001 = ax1.plot(t, phi_msn*1000, 'k')[0]
l003 = ax1.plot(t, phi_msg*1000, '#9467bd')[0]
ax1.set_ylabel('$\phi\mathrm{_{ms}}$ [mV]')
ax1.set_yticks([-70, 0])
ax1.set_xticks([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
ax1.set_xlim(0,10)

### Panel B ###
l01 = ax2.plot(t, phi_msn*1000, 'k', label='neuron')[0]
l02 = ax2.plot(t, phi_msg*1000, '#9467bd', label='glia')[0]
ax2.set_ylabel('$\phi\mathrm{_{ms}}$ [mV]')
ax2.set_yticks([-70, 0])

##### Panel C ###
l1 = ax3.plot(t, Na_se-Na_se[0], zorder=10)[0]
l2 = ax3.plot(t, K_se-K_se[0], zorder=10)[0]
l3 = ax3.plot(t, Cl_se-Cl_se[0], zorder=10)[0]
l4 = ax3.plot(t, Ca_se-Ca_se[0], zorder=10)[0]
ax3.set_ylabel('$\Delta \mathrm{[k]_{se}}$ [mM]')
fig.legend([l01, l02, l1, l2, l3, l4], ['neuron', 'glia', '$\mathrm{Na^+}$', '$\mathrm{K^+}$', '$\mathrm{Cl^-}$', '$\mathrm{Ca^{2+}}$'], \
    loc=(0.60,0.73), ncol=3, fontsize='small', handlelength=0.8, handletextpad=0.4, columnspacing=0.4)
ax3.set_ylim(-60,60)

#### Panel D ###
ax4.plot(t, Na_de-Na_de[0], zorder=10)
ax4.plot(t, K_de-K_de[0], zorder=10)
ax4.plot(t, Cl_de-Cl_de[0], zorder=10)
ax4.plot(t, Ca_de-Ca_de[0], zorder=10)
ax4.set_ylabel('$\Delta \mathrm{[k]_{de}}$ [mM]')
ax4.set_ylim(-60,60)

### Panel E ###
ax5.plot(t, ((V_sn+V_dn)-(V_sn[0]+V_dn[0]))/(V_sn[0]+V_dn[0])*100, 'k', label='neuron')
ax5.plot(t, ((V_se+V_de)-(V_se[0]+V_de[0]))/(V_se[0]+V_de[0])*100, 'k:', label='ECS')
ax5.plot(t, ((V_sg+V_dg)-(V_sg[0]+V_dg[0]))/(V_sg[0]+V_dg[0])*100, 'k--', label='glia')
ax5.set_ylabel('$\Delta V$ [\%]')
ax5.legend(fontsize='small', handlelength=0.8, handletextpad=0.4, loc='lower right', labelspacing=0.01)
ax5.set_yticks([-100, -50, 0, 50])

for ax in [ax2, ax3, ax4, ax5]:
    ax.set_xlim(0,800)
for ax in [ax1, ax2, ax3, ax4, ax5]:
    ax.spines['top'].set_visible(False)
    ax.spines['right'].set_visible(False)

ax5.set_xlabel('time [s]')

# ABC
panel = np.array(['A', 'B', 'C', 'D', 'E'])
i = 0
for ax in [ax1, ax2, ax3, ax4, ax5]:
    ax.text(-0.05, 1.2, panel[i], transform=ax.transAxes, fontsize=12, fontweight='bold', va='top', ha='right')
    i += 1

plt.tight_layout()
plt.savefig('figure4.pdf', dpi=600)

reason: import numpy


text:
import os
import argparse
import subprocess
import numpy as np
import pandas as pd

def training_model(filename, technique, pruning_rate, layer):

    """ Training the pruned model """

    # Opens the temporary file
    f = open('../eval.txt', 'a+')

    # Training with pre-trained weights
    if technique.upper() != 'FROM-SCRATCH':
        weights = filename + str(pruning_rate) + '.conv.' + str(layer)
        command = './darknet detector train ' + filename + '.data ' + filename + '.cfg ' + weights + ' -dont_show -map'
    # Training from scratch
    else:
        command = './darknet detector train ' + filename + '.data ' + filename + '.cfg -dont_show -map'

    # Running training algorithm and saving results to temporary file
    subprocess.call(command, shell = True, stdout = f)

    # Closing file
    f.close()

    print('\n[DONE] Fine-tuning of the model pruned by ' + str(pruning_rate) + '%\n')

def pre_weights(filename, pruning_rate, layer):

    """ Generates a pre-weight from a trained weight """

    # Opens the temporary file
    f = open('../eval.txt', 'a+')

    # Running freezing algorithm and saving results to temporary file
    weights = filename + str(pruning_rate) + '.conv.' + str(layer)
    command = './darknet partial ' + filename + '.cfg ' + filename + '.weights ' + weights + ' ' + str(layer)
    subprocess.call(command, shell = True, stdout = f)

    # Closing file
    f.close()

    print('\n[DONE] Pre-weights of the model pruned by ' + str(pruning_rate) + '%\n')

def valid_set(filename, set):

    """ Changes validation set in the .data file """

    # Opens the file in read-only mode
    f = open(filename + '.data', 'r')

    # Reads lines until EOF
    lines = f.readlines()

    # Loop over the lines
    for i, line in enumerate(lines):
        if 'valid' in line:
          lines[i] = 'valid = data/' + filename + '_' + set + '.txt\n'

    # Opens the file in write-only mode
    f = open(filename + '.data', 'w')

    # Changing validation set in the data file
    f.writelines(lines)
    # Closing file
    f.close()

def hyperparams(filename, img_size, iter, lr, steps):

    """ Changes hyperparameters of the .cfg file """

    # Opens the file in read-only mode
    f = open(filename + '.cfg', 'r')

    # Read lines until EOF
    lines = f.readlines()

    # Loop over the lines
    for i, line in enumerate(lines):
        if 'width' in line:
            lines[i] = 'width = ' + str(img_size) + '\n'
        elif 'height' in line:
            lines[i] = 'height = ' + str(img_size) + '\n'
        elif 'steps = ' in line:
            lines[i] = 'steps = ' + steps + '\n'
        elif 'learning_rate' in line:
            lines[i] = 'learning_rate = ' + str(lr) + '\n'
        elif 'max_batches' in line:
            lines[i] = 'max_batches = ' + str(iter) + '\n'
        elif '^batch =' in line:
            lines[i] = 'batch = 64\n'
        elif '^subdivisions' in line:
            lines[i] = 'subdivisions = 16\n'

    # Opens the file in write-only mode
    f = open(filename + '.cfg', 'w')

    # Changing image size in the config file
    f.writelines(lines)
    # Closing file
    f.close()

if __name__ == '__main__':

    parser = argparse.ArgumentParser()

    parser.add_argument('--method', type = str, help = 'Pruning method')
    parser.add_argument('--imgsize', type = int, default = 416, help = 'Image size')
    parser.add_argument('--dataset', type = str, default = 'dfire', help = 'Dataset name')
    parser.add_argument('--lr', type = float, default = 0.01, help = 'Learning rate')
    parser.add_argument('--tuning-iter', type = int, default = 30000, help = 'Number of fine-tuning iterations')
    parser.add_argument('--layer', type = int, default = 161, help = 'Weights frozen up to this layer for fine-tuning')
    parser.add_argument('--steps', type = str, default = '24000,27000', help = 'At these iterations the learning rate will be multiplied by scales factor (0.1 by default)')
    opt = parser.parse_args()

    # Open root folder
    root = 'Fine-Tuning/' + opt.method + os.sep
    os.chdir(root)

    # Pruned models with pruning rate from 10% to 90%
    folders = np.arange(start = 10, stop = 100, step = 10)
    # Partial weights
    weights = np.arange(start = 1000, stop = opt.tuning_iter + 1000, step = 1000)

    try:
        # Remove current eval.txt file
        os.remove('eval.txt')
    except:
        pass

    for folder in folders:

        # Open current folder
        subdir = str(folder) + os.sep
        os.chdir(subdir)

        # Change hyperparameters
        hyperparams(opt.dataset, opt.imgsize, opt.tuning_iter, opt.lr, opt.steps)
        # Change validation set
        valid_set(opt.dataset, 'valid')
        # Pre-trained weights
        if opt.method.upper() != 'FROM-SCRATCH':
            # Freezing layers to generate pre-weights
            pre_weights(opt.dataset, folder, opt.layer)

        # Training pruned model
        training_model(opt.dataset, opt.method, folder, opt.layer)

        # Remove partial weights
        os.chdir('weights/')
        for w in weights:
            try:
                os.remove(opt.dataset + '_' + str(w) + '.weights')
            except:
                pass
        os.remove(opt.dataset + '_last.weights')
        os.remove(opt.dataset + '_final.weights')

        # Returns to root folder
        os.chdir('../../')

max_stars_repo_name: pedbrgs/Pruning-Darknet

max_issues_repo_name: pedbrgs/Pruning-Darknet

max_forks_repo_name: pedbrgs/Pruning-Darknet


text:
#!/usr/bin/env python3

#  Base class for all geometry in the simulation (the hand, the object, and the environment)
#    The assumption is that all geometry has at least
#       The raw geometry (the stl/obj/mesh) - in the form of vertex locations
#       One (or more) transforms
#           xml/urdf: the transform that is applied to the raw geometry as it is read in from the file
#              - this doesn't change
#           simulation matrix/location: The transform applied to the geometry by the simulation/IK solver, etc
#   In addition, define some useful debugging/calculation tools
#     A bounding box around the original mesh geometry
#     A signed distance function for the original mesh geometry

from numpy import array as nparray
from numpy import eye
from coordinate_system import CoordinateSystemTransform, BoundingBox
from bounding_box import BoundingBox
from signed_distance_fc import SignedDistanceFc
from data_directories_base import DataDirectoryBase

class GeometryBase(DataDirectoryBase):
    # If eg PyBullet or Mujoco or RViz is up and running
    sym_environment = None

    def __init__(self, cls_name, type_name, instance_name=""):
        """Created from an stl file, urdf, xml, etc
             use set functions to create geometry
             Actual geometry can be stored here or in the simulation (if running)"""
        super(DataDirectoryBase, self).__init__(cls_name, instance_name)
        self.name = instance_name
        # bounding box will be in self.mesh_to_world
        self.mesh_to_world = CoordinateSystemTransform((type_name, "origin"), (type_name, "world"))
        self.sdf = SignedDistanceFc()

        # If we're in the simulation, use this to get/set the geometry from the simulation
        self.sim_ref = None

    def __str__(self):
        """TODO """
        return "Geometry"

    def __repr__(self):
        return self.__str__()

    def get_vertices(self):
        """ Generator function that returns all vertices in the form of a 1x4 numpy array [x,y,z,1]
              Suitable for multiplying by a matrix
            Uses the sim_ref pointer """

        if not self.sim_ref:
            from numpy import random
            fake_data = random.rand(10, 3)
            for d in fake_data:
                yield nparray([d[0], d[1], d[2], 1])

    def get_from_bbox(self):
        """Just to make it easier to find
        @returns BoundingBox"""
        return self.mesh_to_world.bbox_from()

    def get_mesh_to_world_matrix(self):
        """Just to make it easier to find
        @returns nparray matrix"""
        return self.mesh_to_world.get_matrix()

    def get_world_to_mesh_matrix(self):
        """Just to make it easier to find
        @returns nparray matrix"""
        return self.mesh_to_world.get_inverse_matrix()

    def set_world_xform(self):
        """ Get the current world transform(s) from the simulator """
        raise NotImplementedError

    def set_signed_distance_fc(self):
        """ Once the geometry is set-up, call this"""
        raise NotImplementedError

    def render(self, draw_mesh_bbox=True):
        """TODO Draw geometry in Open GL
        @param draw_mesh_bbox Draw the mesh bounding box as well"""
        raise NotImplementedError

if __name__ == "__main__":

    my_obj = GeometryBase("test")

max_stars_repo_name: OSUrobotics/KinovaGrasping

max_issues_repo_name: OSUrobotics/KinovaGrasping

max_forks_repo_name: OSUrobotics/KinovaGrasping

reason: from numpy


text:
import unittest

import jax
import jax.numpy as jnp
import numpy as np
import optax

from flax import linen as nn
from flax.training import train_state

class TestFlax(unittest.TestCase):

    def test_pooling(self):
        x = jnp.full((1, 3, 3, 1), 2.)
        mul_reduce = lambda x, y: x * y
        y = nn.pooling.pool(x, 1., mul_reduce, (2, 2), (1, 1), 'VALID')
        np.testing.assert_allclose(y, np.full((1, 2, 2, 1), 2. ** 4))

    def test_cnn(self):
        class CNN(nn.Module):
            @nn.compact
            def __call__(self, x):
                x = nn.Conv(features=32, kernel_size=(3, 3))(x)
                x = nn.relu(x)
                x = nn.avg_pool(x, window_shape=(2, 2), strides=(2, 2))
                x = nn.Conv(features=64, kernel_size=(3, 3))(x)
                x = nn.relu(x)
                x = nn.avg_pool(x, window_shape=(2, 2), strides=(2, 2))
                x = x.reshape((x.shape[0], -1))
                x = nn.Dense(features=256)(x)
                x = nn.relu(x)
                x = nn.Dense(features=120)(x)
                x = nn.log_softmax(x)
                return x

        def create_train_state(rng, learning_rate, momentum):
            cnn = CNN()
            params = cnn.init(rng, jnp.ones([1, 224, 224, 3]))['params']
            tx = optax.sgd(learning_rate, momentum)
            return train_state.TrainState.create(
            apply_fn=cnn.apply, params=params, tx=tx)

        rng = jax.random.PRNGKey(0)
        rng, init_rng = jax.random.split(rng)

        learning_rate = 2e-5
        momentum = 0.9
        state = create_train_state(init_rng, learning_rate, momentum)
        self.assertEqual(0, state.step)

max_stars_repo_name: mcx/docker-python

max_issues_repo_name: mcx/docker-python

max_forks_repo_name: mcx/docker-python

reason: import numpy,import jax


text:
import gym
import numpy as np
import random

from gym import error, spaces, utils
from gym.utils import seeding
from jaco_gym.envs.ros_scripts.jaco_gazebo_action_client import JacoGazeboActionClient

class JacoEnv(gym.Env):

    def __init__(self):

        self.robot = JacoGazeboActionClient()

        self.action_dim = 6
        # self.obs_dim = 36
        self.obs_dim = 12   # when using read_state_simple

        high = np.ones([self.action_dim])
        self.action_space = gym.spaces.Box(-high, high)

        high = np.inf * np.ones([self.obs_dim])
        self.observation_space = gym.spaces.Box(-high, high)

    def convert_action_to_deg(self, a, OldMin, OldMax, NewMin, NewMax):

        OldRange = (OldMax - OldMin)
        NewRange = (NewMax - NewMin)

        return (((a - OldMin) * NewRange) / OldRange) + NewMin

    def action2deg(self, action):
        action[0] = self.convert_action_to_deg(action[0], OldMin=-1, OldMax=1, NewMin=0, NewMax=360)
        action[1] = 180
        action[2] = self.convert_action_to_deg(action[2], OldMin=-1, OldMax=1, NewMin=90, NewMax=270)
        action[3] = self.convert_action_to_deg(action[3], OldMin=-1, OldMax=1, NewMin=0, NewMax=360)
        action[4] = self.convert_action_to_deg(action[4], OldMin=-1, OldMax=1, NewMin=0, NewMax=360)
        action[5] = self.convert_action_to_deg(action[5], OldMin=-1, OldMax=1, NewMin=0, NewMax=360)

        return action

    def step(self, action):

        # convert action from range [-1, 1] to [0, 360]
        self.action = self.action2deg(action)

        # convert to radians
        self.action = np.radians(self.action)

        # move arm
        self.robot.move_arm(self.action)

        # get state
        # self.observation = self.robot.read_state()
        self.observation = self.robot.read_state_simple()   # only return 12 values instead of 36

        # calculate reward
        self.tip_coord = self.robot.get_tip_coord()
        self.dist_to_target = np.linalg.norm(self.tip_coord - self.target_vect)
        self.reward = - self.dist_to_target

        # create info
        self.info = {"tip coordinates": self.tip_coord, "target coordinates": self.target_vect}

        # create done
        self.done = False

        # IF DEFINING DONE AS FOLLOWS, THE EPISODE ENDS EARLY AND A GOOD AGENT WILL RECEIVED A PENALTY FOR BEING GOOD
        # COOMENT THIS
        # if self.dist_to_target < 0.01:
            # self.done = True

        # print("tip position: ", self.tip_coord)
        # print("target vect: ", self.target_vect)
        # print("dist_to_target: ", self.dist_to_target)

        return self.observation, self.reward, self.done, self.info

    def reset(self):

        self.robot.cancel_move()

        pos = [0, 180, 180, 0, 0, 0]
        pos = np.radians(pos)
        self.robot.move_arm(pos)
        print("Jaco reset to initial position")

        # get observation
        # self.obs = self.robot.read_state()
        self.obs = self.robot.read_state_simple()

        # generate random coordinates of a point in space
        # limits of real robot
        # x_target = random.uniform(-0.49, 0.49)
        # y_target = random.uniform(-0.49, 0.49)
        # z_target = random.uniform(0.69, 1.18)

        # limits in Gazebo
        x_target = random.uniform(-0.335, 0.335)
        y_target = random.uniform(-0.337, 0.337)
        z_target = random.uniform(0.686, 1.021)

        self.target_vect = np.array([x_target, y_target, z_target])
        print("Random target coordinates generated")

        # if testing: graphically move the sphere target, if training, comment this line
        # self.robot.move_sphere(self.target_vect)

        return self.obs

    def render(self, mode='human', close=False):
        pass


text:
# Module containing the experimental features of the framework that are still under development.
#
# Author: Fernando García Gutiérrez
# Email: fegarc05@ucm.es
#
import numpy as np
import sys
import random
import warnings
from copy import deepcopy

from ..report import Report
from ..ant import fixedPositions, deleteInitialPosition
from ..base import DecaySchedule
from ..aco import ACO, evaluateAnts, generatePathsACS
from ..algorithm import updateReportWithBest

warnings.filterwarnings('error')

def antColonySystem(aco_obj: ACO, scores_decay: DecaySchedule = None,
                    evaporation_decay: DecaySchedule = None, report: Report = None,
                    save_pheromones: bool = True, verbose: bool = True) -> Report:
    """
    Function that executes the Ant Colony System algorithm based on

        Dorigo, M., & Gambardella, L. M. (1997). Ant colony system: a cooperative learning
        approach to the traveling salesman problem. IEEE Transactions on evolutionary computation,
        1(1), 53-66.

    Parameters
    ----------
    aco_obj: antco.ACO
        Initialised antco.ACO instance, for more information on this type of object use:
        help(antco.ACO)

    scores_decay: antco.optim.DecaySchedule subclass, default=None
        Instance of a antco.optim.DecaySchedule representing a decay schedule that will applied to
        the score values with which the pheromone values are updated in each iteration.
        For more info use help(antco.optim.DecaySchedule).

    evaporation_decay: antco.optim.DecaySchedule subclass, default=None
        Same as scores_decay, in this case used to update the evaporation rate value.
        For more info use help(antco.optim.DecaySchedule).

    report: antco.report.Report, default=None
        antco.report.Report instance, by default a report will be initialised.

    save_pheromones: bool, default=True
        Value indicating whether to store in the report the values of the pheromone matrix at each
        iteration. If True these may be retrieved by:

            >>> report.get('pheromones')

    verbose: bool, default=True
        Indicates whether to show convergence information during the execution of the algorithm.

    Returns
    -------
    :antco.report.Report
        Returns a antco.report.Report instance with information about the convergence of the
        algorithm. The convergence parameters can be accessed via:

        >>> report = antco.algorithm.basic(...)
        >>> report.get('min_cost')
        >>> report.get('mean_cost')
        >>> report.get('max_cost')

        For more info about Report instances use help(antco.report.Report).
    """
    # Get parameters from aco_obj
    seed = aco_obj.seed
    graph = aco_obj.graph
    iterations = aco_obj.iterations
    n_jobs = aco_obj.n_jobs
    ants = aco_obj.ants
    H = aco_obj.heuristic
    P = aco_obj.pheromones
    alpha = aco_obj.alpha
    beta = aco_obj.beta
    if aco_obj.precompute_heuristic:
        H = np.power(H, beta)
    pheromone_update_kw = aco_obj.pheromone_update_kw
    rho = pheromone_update_kw['rho']; del pheromone_update_kw['rho']  # Get evaporation parameter
    decay = pheromone_update_kw['decay']; del pheromone_update_kw['decay']   # Get decay parameter
    pher_init_val = aco_obj.pher_init_val
    updatePheromones = aco_obj.updatePheromones
    objectiveFunction = aco_obj.objectiveFunction
    tol = aco_obj.tol
    fixed = aco_obj.fixed_positions
    accessory_node = aco_obj.accessory_node
    Q = aco_obj.Q
    scaleScores = aco_obj.scaleScores

    if report is None:  # Initialize a Report instance (if not provided)
        report = Report({'BranchingFactor': {'lambda_values': [0.5]}})

    if seed is not None:  # Set random state
        random.seed(seed)
        np.random.seed(seed)

    if fixed and accessory_node is None:  # Put ants in fixed positions (if specified)
        fixedPositions(ants, graph)

    # If an accessory node has been specified, place the ants on that node
    if accessory_node is not None:
        for ant in ants:
            ant.setInitialPosition(accessory_node)

    # Pre-compute the array to store the ant scores
    ant_scores, norm_scores = np.empty(shape=len(ants)), np.empty(shape=len(ants))

    current_iteration = 1
    it_without_improvements = 0
    best_score = -float('inf')
    while current_iteration <= iterations:

        ants = generatePathsACS(   # Graph exploration
            ants=ants, graph=graph, H=H, P=P, alpha=alpha, beta=beta, decay=decay,
            pher_init=pher_init_val, Q=Q, exp_heuristic=False)

        # Evaluate ant paths using the objectiveFunction function (it will be maximized)
        ant_scores = evaluateAnts(ants=ants, objectiveFunction=objectiveFunction)

        # Update best score and save best solution
        new_best = updateReportWithBest(
            ants=ants, scores=ant_scores, best_score=best_score, report=report,
            iteration=current_iteration)

        if new_best > best_score:
            best_score = new_best
            it_without_improvements = 0
        else:
            it_without_improvements += 1

        # Scale scores
        if scaleScores is not None:
            norm_scores[:] = ant_scores[:]  # Copy scores
            norm_scores = scaleScores(norm_scores, best_score)
        else:
            norm_scores = ant_scores

        # Update pheromones according to the scores
        updatePheromones(
            paths=np.array([ant.adj_matrix for ant in ants], dtype=np.int8), P=P,
            ant_scores=norm_scores if scores_decay is None else norm_scores * scores_decay(current_iteration),
            rho=rho if evaporation_decay is None else rho * evaporation_decay(current_iteration),
            **pheromone_update_kw)

        if not fixed and accessory_node is None:  # Restart ants initial position
            deleteInitialPosition(ants)

        # Compute convergence statistics
        mean_scores = np.mean(ant_scores)
        min_score = np.min(ant_scores)
        max_score = np.max(ant_scores)
        report.save(current_iteration, mean_cost=mean_scores, max_cost=max_score)

        if save_pheromones:  # Save pheromone values
            report.save(current_iteration, pheromones=deepcopy(P))

        # Compute monitoring computeMetrics
        report.computeMetrics(current_iteration, P, graph)

        # After several generations without improvements, do an early stopping of the algorithm.
        if it_without_improvements > tol: break

        if verbose:
            sys.stdout.write('\rCost: Mean: %.4f (Min: %.4f Max: %.4f) (iteration %d)' %
                             (float(mean_scores), min_score, max_score, current_iteration))

        current_iteration += 1

    return report

max_stars_repo_name: FernandoGaGu/Ant-Colony-Optimisation

max_issues_repo_name: FernandoGaGu/Ant-Colony-Optimisation

max_forks_repo_name: FernandoGaGu/Ant-Colony-Optimisation


text:
import gym
import numpy as np
from keras.layers import Activation, Dense, Flatten
from keras.models import Sequential
from keras.optimizers import Adam
from rl.agents.dqn import DQNAgent
from rl.memory import SequentialMemory
from rl.policy import EpsGreedyQPolicy

ENV_NAME = 'CartPole-v0'

env = gym.make(ENV_NAME)
np.random.seed(123)
env.seed(123)
nb_actions = env.action_space.n

model = Sequential()
model.add(Flatten(input_shape=(1,) + env.observation_space.shape))
model.add(Dense(16))
model.add(Activation('relu'))
model.add(Dense(nb_actions))
model.add(Activation('linear'))
print(model.summary())

policy = EpsGreedyQPolicy()
memory = SequentialMemory(limit=50000, window_length=1)
dqn = DQNAgent(model=model, nb_actions=nb_actions, memory=memory, nb_steps_warmup=10, target_model_update=1e-2, policy=policy)
dqn.compile(Adam(lr=1e-3), metrics=['mae'])
dqn.fit(env, nb_steps=5000, visualize=True, verbose=2)
dqn.test(env, nb_episodes=5, visualize=True)

max_stars_repo_name: cgnerds/UnityML

max_issues_repo_name: cgnerds/UnityML

max_forks_repo_name: cgnerds/UnityML


text:
import numpy as np
from random import shuffle
from past.builtins import xrange

def softmax_loss_naive(W, X, y, reg):
  """
  Softmax loss function, naive implementation (with loops)

  Inputs have dimension D, there are C classes, and we operate on minibatches
  of N examples.

  Inputs:
  - W: A numpy array of shape (D, C) containing weights.
  - X: A numpy array of shape (N, D) containing a minibatch of data.
  - y: A numpy array of shape (N,) containing training labels; y[i] = c means
    that X[i] has label c, where 0 <= c < C.
  - reg: (float) regularization strength

  Returns a tuple of:
  - loss as single float
  - gradient with respect to weights W; an array of same shape as W
  """
  # Initialize the loss and gradient to zero.
  loss = 0.0
  dW = np.zeros_like(W)
  y_one_hot = np.zeros((y.shape[0],W.shape[1]))
  y_one_hot[np.arange(y.shape[0]), y] = 1
  #############################################################################
  # TODO: Compute the softmax loss and its gradient using explicit loops.     #
  # Store the loss in loss and the gradient in dW. If you are not careful     #
  # here, it is easy to run into numeric instability. Don't forget the        #
  # regularization!                                                           #
  #############################################################################
  for i in range(X.shape[0]):
    scores = X[i].dot(W)
    scores -= np.max(scores)
    scores = np.exp(scores) / np.sum(np.exp(scores), axis=0)
    # deriv(softmax) = scores*(1-scores)

    for j in range(y_one_hot.shape[1]):
        loss += - y_one_hot[i,j]*(np.log(scores[j]))
        dW[:,j] += X[i]*(scores[j] - y_one_hot[i,j] )

  loss = loss / X.shape[0]
  loss += reg* np.sum(W*W)
  dW += reg* 2 * W
  dW /= X.shape[0]
  #############################################################################
  #                          END OF YOUR CODE                                 #
  #############################################################################

  return loss, dW

def softmax_loss_vectorized(W, X, y, reg):
  """
  Softmax loss function, vectorized version.

  Inputs and outputs are the same as softmax_loss_naive.
  """
  # Initialize the loss and gradient to zero.
  loss = 0.0
  dW = np.zeros_like(W)

  #############################################################################
  # TODO: Compute the softmax loss and its gradient using no explicit loops.  #
  # Store the loss in loss and the gradient in dW. If you are not careful     #
  # here, it is easy to run into numeric instability. Don't forget the        #
  # regularization!                                                           #
  #############################################################################
  scores = X.dot(W)
  scores -= np.max(scores,axis=1, keepdims=True)
  scores_exp = np.exp(scores)/ np.sum(np.exp(scores), axis=1)[:,np.newaxis]
  y_one_hot = np.zeros((y.shape[0],W.shape[1]))
  y_one_hot[np.arange(y.shape[0]), y] = 1

  scores_exp_fil = scores_exp * y_one_hot
  scores_exp_fil = np.ravel(scores_exp_fil)
  scores_exp_fil = scores_exp_fil [scores_exp_fil>0]

  loss = -np.sum(np.log(scores_exp_fil))
  loss = loss / X.shape[0]
  loss += reg* np.sum(W*W)

  dscores = scores_exp
  dscores[np.arange(X.shape[0]),y] -= 1 # this is equal to (scores[j] - y_one_hot[i,j] )

  dW = X.T.dot(dscores) / X.shape[0]

  dW += reg * 2 * W
  #############################################################################
  #                          END OF YOUR CODE                                 #
  #############################################################################

  return loss, dW

max_stars_repo_name: radpet/stuff

max_issues_repo_name: radpet/stuff

max_forks_repo_name: radpet/stuff


text:
import ctypes
import numpy
from numpy.ctypeslib import ndpointer
import pkg_resources
import enum
import os
import platform
import struct
import json

from nptyping import NDArray

from brainflow.board_shim import BrainFlowError, LogLevels
from brainflow.exit_codes import BrainflowExitCodes

class BrainFlowMetrics(enum.IntEnum):
    """Enum to store all supported metrics"""

    RELAXATION = 0  #:
    CONCENTRATION = 1  #:
    USER_DEFINED = 2  #:

class BrainFlowClassifiers(enum.IntEnum):
    """Enum to store all supported classifiers"""

    REGRESSION = 0  #:
    KNN = 1  #:
    SVM = 2  #:
    LDA = 3  #:
    DYN_LIB_CLASSIFIER = 4  #:

class BrainFlowModelParams(object):
    """ inputs parameters for prepare_session method

    :param metric: metric to calculate
    :type metric: int
    :param classifier: classifier to use
    :type classifier: int
    :param file: file to load model
    :type file: str
    :param other_info: additional information
    :type other_info: int
    """

    def __init__(self, metric, classifier) -> None:
        self.metric = metric
        self.classifier = classifier
        self.file = ''
        self.other_info = ''

    def to_json(self) -> None:
        return json.dumps(self, default=lambda o: o.__dict__,
                          sort_keys=True, indent=4)

class MLModuleDLL(object):
    __instance = None

    @classmethod
    def get_instance(cls):
        if cls.__instance is None:
            cls.__instance = cls()
        return cls.__instance

    def __init__(self):
        if platform.system() == 'Windows':
            if struct.calcsize("P") * 8 == 64:
                dll_path = 'lib\\MLModule.dll'
            else:
                dll_path = 'lib\\MLModule32.dll'
        elif platform.system() == 'Darwin':
            dll_path = 'lib/libMLModule.dylib'
        else:
            dll_path = 'lib/libMLModule.so'
        full_path = pkg_resources.resource_filename(__name__, dll_path)
        if os.path.isfile(full_path):
            # for python we load dll by direct path but this dll may depend on other dlls and they will not be found!
            # to solve it we can load all of them before loading the main one or change PATH\LD_LIBRARY_PATH env var.
            # env variable looks better, since it can be done only once for all dependencies
            dir_path = os.path.abspath(os.path.dirname(full_path))
            if platform.system() == 'Windows':
                os.environ['PATH'] = dir_path + os.pathsep + os.environ.get('PATH', '')
            else:
                os.environ['LD_LIBRARY_PATH'] = dir_path + os.pathsep + os.environ.get('LD_LIBRARY_PATH', '')
            self.lib = ctypes.cdll.LoadLibrary(full_path)
        else:
            raise FileNotFoundError(
                'Dynamic library %s is missed, did you forget to compile brainflow before installation of python package?' % full_path)

        self.set_log_level_ml_module = self.lib.set_log_level_ml_module
        self.set_log_level_ml_module.restype = ctypes.c_int
        self.set_log_level_ml_module.argtypes = [
            ctypes.c_int
        ]

        self.set_log_file_ml_module = self.lib.set_log_file_ml_module
        self.set_log_file_ml_module.restype = ctypes.c_int
        self.set_log_file_ml_module.argtypes = [
            ctypes.c_char_p
        ]

        self.prepare = self.lib.prepare
        self.prepare.restype = ctypes.c_int
        self.prepare.argtypes = [
            ctypes.c_char_p
        ]

        self.release = self.lib.release
        self.release.restype = ctypes.c_int
        self.release.argtypes = [
            ctypes.c_char_p
        ]

        self.release_all = self.lib.release_all
        self.release_all.restype = ctypes.c_int
        self.release_all.argtypes = []

        self.predict = self.lib.predict
        self.predict.restype = ctypes.c_int
        self.predict.argtypes = [
            ndpointer(ctypes.c_double),
            ctypes.c_int,
            ndpointer(ctypes.c_double),
            ctypes.c_char_p
        ]

class MLModel(object):
    """MLModel class used to calc derivative metrics from raw data

    :param model_params: Model Params
    :type model_params: BrainFlowModelParams
    """

    def __init__(self, model_params: BrainFlowModelParams) -> None:
        self.model_params = model_params
        try:
            self.serialized_params = model_params.to_json().encode()
        except BaseException:
            self.serialized_params = model_params.to_json()

    @classmethod
    def _set_log_level(cls, log_level: int) -> None:
        """set BrainFlow log level, use it only if you want to write your own messages to BrainFlow logger,
        otherwise use enable_ml_logger, enable_dev_ml_logger or disable_ml_logger

        :param log_level: log level, to specify it you should use values from LogLevels enum
        :type log_level: int
        """
        res = MLModuleDLL.get_instance().set_log_level_ml_module(log_level)
        if res != BrainflowExitCodes.STATUS_OK.value:
            raise BrainFlowError('unable to enable logger', res)

    @classmethod
    def enable_ml_logger(cls) -> None:
        """enable ML Logger with level INFO, uses stderr for log messages by default"""
        cls._set_log_level(LogLevels.LEVEL_INFO.value)

    @classmethod
    def disable_ml_logger(cls) -> None:
        """disable BrainFlow Logger"""
        cls._set_log_level(LogLevels.LEVEL_OFF.value)

    @classmethod
    def enable_dev_ml_logger(cls) -> None:
        """enable ML Logger with level TRACE, uses stderr for log messages by default"""
        cls._set_log_level(LogLevels.LEVEL_TRACE.value)

    @classmethod
    def set_log_file(cls, log_file: str) -> None:
        """redirect logger from stderr to file, can be called any time

        :param log_file: log file name
        :type log_file: str
        """
        try:
            file = log_file.encode()
        except BaseException:
            file = log_file
        res = MLModuleDLL.get_instance().set_log_file_ml_module(file)
        if res != BrainflowExitCodes.STATUS_OK.value:
            raise BrainFlowError('unable to redirect logs to a file', res)

    @classmethod
    def release_all(cls) -> None:
        """release all classifiers"""

        res = MLModuleDLL.get_instance().release_all()
        if res != BrainflowExitCodes.STATUS_OK.value:
            raise BrainFlowError('unable to release classifiers', res)

    def prepare(self) -> None:
        """prepare classifier"""

        res = MLModuleDLL.get_instance().prepare(self.serialized_params)
        if res != BrainflowExitCodes.STATUS_OK.value:
            raise BrainFlowError('unable to prepare classifier', res)

    def release(self) -> None:
        """release classifier"""

        res = MLModuleDLL.get_instance().release(self.serialized_params)
        if res != BrainflowExitCodes.STATUS_OK.value:
            raise BrainFlowError('unable to release classifier', res)

    def predict(self, data: NDArray) -> float:
        """calculate metric from data

        :param data: input array
        :type data: NDArray
        :return: metric value
        :rtype: float
        """
        output = numpy.zeros(1).astype(numpy.float64)
        res = MLModuleDLL.get_instance().predict(data, data.shape[0], output, self.serialized_params)
        if res != BrainflowExitCodes.STATUS_OK.value:
            raise BrainFlowError('unable to calc metric', res)
        return output[0]

max_stars_repo_name: xgdgsc/brainflow

max_issues_repo_name: HDPNC/brainflow

max_forks_repo_name: HDPNC/brainflow


text:
# -*- coding: utf-8 -*-"

import numpy as np
import scipy.stats as stats
from bisect import bisect_left

def quantile_sp(c_, x, p=None, method=None, h=None):
    """For details, see here:

    Parameters
    ----------
        c_ : scalar, array, shape(k_,)
        x : array, shape (j_,)
        p : array, shape (j_,), optional
        method: string, optional
        h: scalar, optional

    Returns
    -------
        q : array, shape (k_,)

    """

    c_ = np.atleast_1d(c_)
    j_ = x.shape[0]
    k_ = c_.shape[0]

    # Step 1: Sorted scenarios-probabilities

    if p is None:
        p = np.ones(j_) / j_  # equal probabilities as default value

    sort_x = np.argsort(x)
    x_sort = x[sort_x]
    p_sort = p[sort_x]

    # Step 2: Cumulative sums of sorted probabilities

    u_sort = np.zeros(j_ + 1)
    for j in range(1, j_ + 1):
        u_sort[j] = np.sum(p_sort[:j])

    # Step 3: Output quantile

    q = np.zeros(k_)

    qindx = [0]*k_
    for k in range(k_):
        qindx[k] = bisect_left(u_sort, c_[k])-1
        if qindx[k] == -1:
            qindx[k] = 0
        if qindx[k] >= x.shape[0]:
            qindx[k] = x.shape[0]-1

    if method == "linear_interp":
        x_0 = x_sort[0]-(x_sort[1]-x_sort[0])*u_sort[1]/(u_sort[2]-u_sort[1])
        x_sort = np.append(x_0, x_sort)
        for k in range(k_):
            q[k] = x_sort[qindx[k]] + \
                       (x_sort[qindx[k]+1] - x_sort[qindx[k]]) *\
                       (c_[k]-u_sort[qindx[k]]) /\
                       (u_sort[qindx[k]+1]-u_sort[qindx[k]])

    elif method == "kernel_smoothing":
        if h is None:
            h = 0.25*(j_**(-0.2))

        for k in range(k_):
            w = np.diff(stats.norm.cdf(u_sort, c_[k], h))
            w = w / np.sum(w)
            q[k] = x_sort @ w

    else:
        q = x_sort[qindx]

    return np.squeeze(q)

max_stars_repo_name: dpopadic/arpmRes

max_issues_repo_name: dpopadic/arpmRes

max_forks_repo_name: dpopadic/arpmRes

reason: import numpy,import scipy


text:
#! /usr/bin/env python

descr   = """
"""

import os
import sys

DISTNAME            = 'openopt'
DESCRIPTION         = 'A python module for numerical optimization'
LONG_DESCRIPTION    = descr
MAINTAINER          = 'Dmitrey Kroshko'
MAINTAINER_EMAIL    = 'dmitrey-at-openopt-dot-org'
URL                 = 'http://openopt.org'
LICENSE             = 'new BSD'

sys.path.append(os.getcwd() + os.sep + 'openopt')
from ooVersionNumber import __version__ as ooVer
openopt_version = ooVer

#DOWNLOAD_URL        = 'http://openopt.org/images/3/33/OpenOpt.zip'

try:
    import setuptools
except:
    print('you should have setuptools installed (http://pypi.python.org/pypi/setuptools), for some Linux distribs you can get it via [sudo] apt-get install python-setuptools')
    print('press Enter for exit...')
    raw_input()
    exit()

#from numpy.distutils.system_info import system_info, NotFoundError, dict_append, so_ext
from numpy.distutils.core import setup

DOC_FILES = []

def configuration(parent_package='',top_path=None, package_name=DISTNAME):
    if os.path.exists('MANIFEST'): os.remove('MANIFEST')
    #pkg_prefix_dir = 'openopt'

    # Get the version

    from numpy.distutils.misc_util import Configuration
    config = Configuration(package_name,parent_package,top_path,
        version     = openopt_version,
        maintainer  = MAINTAINER,
        maintainer_email = MAINTAINER_EMAIL,
        description = DESCRIPTION,
        license = LICENSE,
        url = URL,
        #download_url = DOWNLOAD_URL,
        long_description = LONG_DESCRIPTION)

    # XXX: once in SVN, should add svn version...
    #print config.make_svn_version_py()

    # package_data does not work with sdist for setuptools 0.5 (setuptools bug),
    # so we need to add them here while the bug is not solved...

    return config

if __name__ == "__main__":

    # setuptools version of config script

    # package_data does not work with sdist for setuptools 0.5 (setuptools bug)
    # So we cannot add data files via setuptools yet.

    #data_files = ['test_data/' + i for i in TEST_DATA_FILES]
    #data_files.extend(['docs/' + i for i in doc_files])
    setup(configuration = configuration,
        install_requires=['numpy','setproctitle'], # can also add version specifiers   #namespace_packages=['kernel'],
        #setup_requires = 'setproctitle',
        #py_modules = ['kernel', 'tests', 'examples', 'solvers'],
        packages=setuptools.find_packages(),
        include_package_data = True,
        #package_data = '*.txt',
        test_suite='',#"openopt.tests", # for python setup.py test
        zip_safe=False, # the package can run out of an .egg file
        #FIXME url, download_url, ext_modules
        classifiers =
            [ 'Development Status :: 5 - Production/Stable',
              'Environment :: Console',
              'Intended Audience :: Developers',
              'Intended Audience :: Science/Research',
              'License :: OSI Approved :: BSD License',
              'Topic :: Scientific/Engineering']
    )

max_stars_repo_name: PythonCharmers/OOSuite

max_issues_repo_name: PythonCharmers/OOSuite

max_forks_repo_name: PythonCharmers/OOSuite


text:
#!/usr/bin/env python

# this function has:
#  * a minimum at [x == -1.012273212379936, y == 0.0]
#    (f_xx*f_yy - f_xy**2 > 0 and f_xx > 0)
#  * a saddle point at [x == 0.9872574572835215, y == 0.0]
#    (f_xx*f__y - f_xy**2 < 0)
#  * a maximum at [x == 0.02501565538380409, y == 0.0]
#    (f_xx*f_yy - f_xy**2 > 0 and f_xx < 0)
def func(X, y=None):
    if y is None:
        x, y = X
    else:
        x = X
    return (x**2 + y**2)**2 - 2*x**2 - 2*y**2 + 0.1*x

def func_grad(X):
    x, y = X
    func_x = 4*(x**2 + y**2)*x - 4*x + 0.1
    func_y = 4*(x**2 + y**2)*y - 4*y
    return np.array([func_x, func_y])

def func_hess(X):
    x, y = X
    func_x_x = 12*x**2 + 4*y**2 - 4
    func_x_y = 8*x*y
    func_y_x = 8*x*y
    func_y_y = 4*x**2 + 12*y**2 - 4
    return np.array([[func_x_x, func_x_y], [func_y_x, func_y_y]])

def check_solution(x):
    from scipy.linalg import det
    hessian = func_hess(x)
    det_hessian = det(hessian)
    type_str = '  ||H|| = {0:.4f}'.format(det_hessian)
    if det_hessian > 0.0:
        type_str += ', f_xx = {0:.4f}'.format(hessian[0, 0])
        if hessian[0, 0] > 0.0:
            type_str += ': relative minimum'
        else:
            type_str += ': relative maximum'
    elif det_hessian < 0.0:
        type_str += ': saddle point'
    else:
        type_str += ': inconclusive'
    print(type_str)

if __name__ == '__main__':
    from argparse import ArgumentParser
    import time
    import numpy as np
    import scipy.optimize as opt

# handle command line arguments
    arg_parser = ArgumentParser(description='optimize a 2D-function')
    arg_parser.add_argument('--x0', type=float, default=1.0,
                            help='initial x-value')
    arg_parser.add_argument('--y0', type=float, default=0.0,
                            help='initial y-value')
    arg_parser.add_argument('--xtol', type=float, default=1.0e-4,
                            help='line-search error tolerance (Powell)')
    arg_parser.add_argument('--ftol', type=float, default=1.0e-4,
                            help='relative error in function value '
                                 'acceptable for convergence (Powell)')
    arg_parser.add_argument('--gtol', type=float, default=1.0e-5,
                            help='stop when the norm of the gradient is '
                                 'less than gtol (CG & BFGS)')
    arg_parser.add_argument('--avextol', type=float, default=1.0e-5,
                            help='stop when the average relative error is '
                                 'less than gtol (N-CG)')
    arg_parser.add_argument('--check', action='store_true',
                            help='check critical point')
    options = arg_parser.parse_args()

# Powell's method
    x0 = np.array([options.x0, options.y0])
    start = time.time()
    xopt, fopt, _, iter, funcalls, warnflag = opt.fmin_powell(
                                                  func, x0,
                                                  xtol=options.xtol,
                                                  ftol=options.ftol,
                                                  full_output=True,
                                                  disp=False
                                              )
    durationn = time.time() - start
    output_str = '  minimum {fopt:.15} at ({x:.15}, {y:.15})'
    info_str = '  needed {iter} iterations, {calls} function calls'
    time_str = '  in {sec:.6f} seocnds'
    warn_str = '  result flag = {flag}'
    print("Powell's method:")
    print(output_str.format(fopt=fopt, x=xopt[0], y=xopt[1]))
    print(info_str.format(iter=iter, calls=funcalls))
    print(time_str.format(sec=durationn))
    print(warn_str.format(flag=warnflag))
    if options.check:
        check_solution(xopt)

# conjugate gradient method
    x0 = np.array([options.x0, options.y0])
    start = time.time()
    xopt, fopt, funcalls, gradcalls, warnflag = opt.fmin_cg(
                                                    func, x0,
                                                    fprime=func_grad,
                                                    gtol=options.gtol,
                                                    full_output=True,
                                                    disp=False
                                                )
    durationn = time.time() - start
    output_str = '  minimum {fopt:.15} at ({x:.15}, {y:.15})'
    info_str = '  needed {calls} function calls, {gradcalls} gradient calls'
    time_str = '  in {sec:.6f} seocnds'
    warn_str = '  result flag = {flag}'
    print('\nconjugate gradient method')
    print(output_str.format(fopt=fopt, x=xopt[0], y=xopt[1]))
    print(info_str.format(calls=funcalls, gradcalls=gradcalls))
    print(time_str.format(sec=durationn))
    print(warn_str.format(flag=warnflag))
    if options.check:
        check_solution(xopt)

# Broyden-Fletcher-Goldfarb-Shanno algorithm
    x0 = np.array([options.x0, options.y0])
    start = time.time()
    xopt, fopt, _, _, funcalls, gradcalls, warnflag = opt.fmin_bfgs(
                                                          func, x0,
                                                          fprime=func_grad,
                                                          gtol=options.gtol,
                                                          full_output=True,
                                                          disp=False
                                                      )
    durationn = time.time() - start
    output_str = '  minimum {fopt:.15} at ({x:.15}, {y:.15})'
    info_str = '  needed {calls} function calls, {gradcalls} gradient calls'
    time_str = '  in {sec:.6f} seocnds'
    warn_str = '  result flag = {flag}'
    print('\nBFGS method')
    print(output_str.format(fopt=fopt, x=xopt[0], y=xopt[1]))
    print(info_str.format(calls=funcalls, gradcalls=gradcalls))
    print(time_str.format(sec=durationn))
    print(warn_str.format(flag=warnflag))
    if options.check:
        check_solution(xopt)

# Newton conjugate gradient method
    x0 = np.array([options.x0, options.y0])
    start = time.time()
    xopt, fopt, funcalls, gradcalls, hesscalls, warnflag = opt.fmin_ncg(
                                                    func, x0,
                                                    fprime=func_grad,
                                                    fhess=func_hess,
                                                    avextol=options.avextol,
                                                    full_output=True,
                                                    disp=False
                                                )
    durationn = time.time() - start
    output_str = '  minimum {fopt:.15} at ({x:.15}, {y:.15})'
    info_str = ('  needed {calls} function calls'
                ', {gradcalls} gradient calls'
                ', {hesscalls} Hessian calls')
    time_str = '  in {sec:.6f} seocnds'
    warn_str = '  result flag = {flag}'
    print('\nNewton conjugate gradient method')
    print(output_str.format(fopt=fopt, x=xopt[0], y=xopt[1]))
    print(info_str.format(calls=funcalls, gradcalls=gradcalls,
                          hesscalls=hesscalls))
    print(time_str.format(sec=durationn))
    print(warn_str.format(flag=warnflag))
    if options.check:
        check_solution(xopt)

max_stars_repo_name: Gjacquenot/training-material

max_issues_repo_name: Gjacquenot/training-material

max_forks_repo_name: Gjacquenot/training-material

reason: import numpy,import scipy,from scipy


text:
from matplotlib.axes import SubplotBase
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns

from typing import List

def pie_plot(feature_name: str, df: pd.DataFrame, ax: SubplotBase=None, title: str=None):
    title = title if title is not None else f'{feature_name} Pie Chart'
    if not ax:
        fig, ax = plt.subplots()

    _ = df[feature_name].value_counts(normalize=True).plot.pie(autopct="%.3f%%", labels=['No ER', 'ER'])
    ax.set_title(title)
    ax.set_ylabel(None)
    plt.tight_layout()

def density_bar_plot(feature_name: str, df: pd.DataFrame,  ax: SubplotBase=None, rot:int=0, title: str=None,):
    title = title if title is not None else f'{feature_name} Bar Plot'
    if not ax:
        fig, ax = plt.subplots()

    bar_plot = df[feature_name].value_counts(normalize=True).plot.bar(ax=ax, rot=rot)
    annotate_bar_plot_with_proportions(bar_plot, ax=ax, fontsize=11)

    ax.set_title(title)
    ax.set_ylabel('density')

def stacked_bar_plot(feature_a: str, feature_b: str, df: pd.DataFrame, ax: SubplotBase=None, title: str=None):
    title = title if title is not None else f'{feature_a} vs {feature_b} Stacked Bar Plot'

    if not ax:
        fig, ax = plt.subplots()

    c = pd.crosstab(df[feature_a], df[feature_b]).apply(lambda r: r/r.sum(), axis=1)
    stacked = c.plot.bar(stacked=True, rot=0, ax=ax)

    for p in stacked.patches:
        height = p.get_height()
        stacked.text(p.get_x() + p.get_width() / 2,
              p.get_y() + height / 2,
              "{:.3f}".format(height),
              ha='center',
              va='bottom')

    ax.set_title(title)
    ax.set_ylabel('density')

def grouped_bar_plot(feature_name: str, df: pd.DataFrame, ax: SubplotBase=None, groupby: str='is_readmitted_early', order: list=None, title: str=None, xlabels: list=[], rot:int=0):
    title = title if title is not None else f'Proportion of {feature_name}, grouped by {groupby}'
    y_colname = f'density in {groupby} group'
    grouped = df.groupby(groupby)[feature_name] \
        .value_counts(normalize=True) \
        .rename(y_colname).to_frame() \
        .reset_index() \
        .rename(columns={'level_1': feature_name}) \

    if not ax:
        fig, ax = plt.subplots()
        fig.set_figwidth(10)

    bar_plot = sns.barplot(x=feature_name, y=y_colname, hue=groupby, data=grouped, ax=ax, order=order)

    if xlabels:
        ax.set_xticklabels(xlabels, rotation=rot)
    else:
        labels = ax.get_xticklabels()
        ax.set_xticklabels(labels, rotation=rot)

    ax.set_title(title, pad=20)
    ax.legend(loc='upper left', bbox_to_anchor=(1.025, 1))
    annotate_bar_plot_with_proportions(bar_plot, ax=ax)

def annotate_bar_plot_with_proportions(bar_plot, fontsize=9, vertical=False, ax=None):
    """
    Annotates values atop of each vertical in a bar plot.
    """
    for p in bar_plot.patches:
        if ax:
            ax.annotate(format(p.get_height(), '.3f'), (p.get_x() + p.get_width() / 2., p.get_height()),
                ha = 'center', va = 'center',
                xytext = (2 if vertical else 0, 13 if vertical else 9),
                textcoords='offset points',
                fontsize=fontsize,
                rotation=90 if vertical else 0)
        else:
            plt.annotate(format(p.get_height(), '.3f'), (p.get_x() + p.get_width() / 2., p.get_height()),
                ha = 'center', va = 'center',
                xytext = (2 if vertical else 0, 13 if vertical else 9),
                textcoords='offset points',
                fontsize=fontsize,
                rotation=90 if vertical else 0)

def truncate_xticklabels(plot, stop_index=30):
    """
    Truncate xtick labels with an ellipsis after `stop_index`
    """
    for label in plot.get_xticklabels():
        t = label.get_text()
        if len(t) < stop_index:
            continue
        else:
            label.set_text(t[:stop_index] + '...')

    return plot.get_xticklabels()

def grouped_box_plot(feature_name: str, df: pd.DataFrame, groupby: str, ax: SubplotBase=None):
    if not ax:
        fig, ax = plt.subplots()

    sns.boxplot(y=feature_name, x=groupby, data=df, ax=ax,
        showmeans=True, boxprops=dict(alpha=.7), width=.3,
        flierprops=dict(markersize=3,alpha=0.1,
        linestyle='none'))

max_stars_repo_name: uncanny-valley/diabetes-readmission

max_issues_repo_name: uncanny-valley/diabetes-readmission

max_forks_repo_name: uncanny-valley/diabetes-readmission


text:
import os
import torch
import torch_scatter
import numpy as np
import scipy.io as sio
import openmesh as om

def readOBJ(file):
    V, Vt, F, Ft = [], [], [], []
    with open(file, 'r') as f:
        T = f.readlines()
    for t in T:
        # 3D vertex
        if t.startswith('v '):
            v = [float(n) for n in t.replace('v ','').split(' ')]
            V += [v]
        # UV vertex
        elif t.startswith('vt '):
            v = [float(n) for n in t.replace('vt ','').split(' ')]
            Vt += [v]
        # Face
        elif t.startswith('f '):
            idx = [n.split('/') for n in t.replace('f ','').split(' ')]
            idx = [i for i in idx if i[0]!='']
            f = [int(n[0]) - 1 for n in idx]
            F += [f]
            # UV face
            if '/' in t:
                f = [int(n[1]) - 1 for n in idx]
                Ft += [f]
    V = np.array(V, np.float32)
    Vt = np.array(Vt, np.float32)
    if Ft: assert len(F) == len(Ft), 'Inconsistent .obj file, mesh and UV map do not have the same number of faces'
    else: Vt, Ft = None, None
    return V, F, Vt, Ft

def loadInfo(filename):
    '''
    this function should be called instead of direct sio.loadmat
    as it cures the problem of not properly recovering python dictionaries
    from mat files. It calls the function check keys to cure all entries
    which are still mat-objects
    '''
    data = sio.loadmat(filename, struct_as_record=False, squeeze_me=True)
    del data['__globals__']
    del data['__header__']
    del data['__version__']
    return _check_keys(data)

def _check_keys(dict):
    '''
    checks if entries in dictionary are mat-objects. If yes
    todict is called to change them to nested dictionaries
    '''
    for key in dict:
        if isinstance(dict[key], sio.matlab.mio5_params.mat_struct):
            dict[key] = _todict(dict[key])
    return dict

def _todict(matobj):
    '''
    A recursive function which constructs from matobjects nested dictionaries
    '''
    dict = {}
    for strg in matobj._fieldnames:
        elem = matobj.__dict__[strg]
        if isinstance(elem, sio.matlab.mio5_params.mat_struct):
            dict[strg] = _todict(elem)
        elif isinstance(elem, np.ndarray) and np.any([isinstance(item, sio.matlab.mio5_params.mat_struct) for item in elem]):
            dict[strg] = [None] * len(elem)
            for i,item in enumerate(elem):
                if isinstance(item, sio.matlab.mio5_params.mat_struct):
                    dict[strg][i] = _todict(item)
                else:
                    dict[strg][i] = item
        else:
            dict[strg] = elem
    return dict

def zRotMatrix(zrot):
    c, s = np.cos(zrot), np.sin(zrot)
    return np.array([[c, -s, 0],
                     [s,  c, 0],
                     [0,  0, 1]], np.float32)

def calc_body_mesh_info(body_model):
    ommesh = om.TriMesh()
    om_v_list = []
    for v in body_model.v_template.detach().cpu().numpy().reshape(6890, 3):
        om_v_list.append(ommesh.add_vertex(v))
    for f in body_model.faces.astype(np.int32):
        ommesh.add_face([om_v_list[i] for i in f])
    vf_fid=torch.zeros(0,dtype=torch.long)
    vf_vid=torch.zeros(0,dtype=torch.long)
    for vid,fids in enumerate(ommesh.vertex_face_indices()):
        fids=torch.from_numpy(fids[fids>=0]).to(torch.long)
        vf_fid=torch.cat((vf_fid,fids),dim=0)
        vf_vid=torch.cat((vf_vid,fids.new_ones(fids.shape)*vid),dim=0)
    return vf_fid, vf_vid

def calc_garment_mesh_info(garment_v, garment_f):
    ommesh = om.TriMesh()
    om_v_list = []
    for v in garment_v:
        om_v_list.append(ommesh.add_vertex(v))
    for f in garment_f.astype(np.int32):
        ommesh.add_face([om_v_list[i] for i in f])
    vf_fid=torch.zeros(0,dtype=torch.long)
    vf_vid=torch.zeros(0,dtype=torch.long)
    for vid,fids in enumerate(ommesh.vertex_face_indices()):
        fids=torch.from_numpy(fids[fids>=0]).to(torch.long)
        vf_fid=torch.cat((vf_fid,fids),dim=0)
        vf_vid=torch.cat((vf_vid,fids.new_ones(fids.shape)*vid),dim=0)
    return vf_fid, vf_vid

def compute_fnorms(verts,tri_fs):
    v0=verts.index_select(-2,tri_fs[:,0])
    v1=verts.index_select(-2,tri_fs[:,1])
    v2=verts.index_select(-2,tri_fs[:,2])
    e01=v1-v0
    e02=v2-v0
    fnorms=torch.cross(e01,e02,-1)
    diss=fnorms.norm(2,-1).unsqueeze(-1)
    diss=torch.clamp(diss,min=1.e-6,max=float('inf'))
    fnorms=fnorms/diss
    return fnorms

def compute_vnorms(verts,tri_fs,vertex_index,face_index):
    fnorms=compute_fnorms(verts,tri_fs)
    vnorms=torch_scatter.scatter(fnorms.index_select(-2,face_index),vertex_index,-2,None,verts.shape[-2])
    diss=vnorms.norm(2,-1).unsqueeze(-1)
    diss=torch.clamp(diss,min=1.e-6,max=float('inf'))
    vnorms=vnorms/diss
    return vnorms

def interpolateBarycentricCoorNumpy(v, ind, w):
    indv = v[ind].reshape(ind.shape[0], 3, 3)
    rv = np.matmul(w.reshape(w.shape[0], 1, 3), indv).reshape(w.shape[0], 3)
    return rv


text:
import tensorflow as tf
import numpy as np
import keras
from keras import backend as K
from keras.layers import Layer, Conv2D
from tensorflow.python.keras import activations

class NormAdd(Layer):

     def __init__(self, activation=None, **kwargs):
          self._initial_weights = kwargs.pop('weights', None)
          self.kwargs = kwargs
          self.activation = activations.get(activation)
          self.concat = keras.layers.Concatenate(axis=-1)
          super(NormAdd, self).__init__(**kwargs)

     def build(self, input_shape):
          n = len(input_shape)
          self.filters = input_shape[0][-1]

          # Weights
          if self._initial_weights is not None:
               weights_conv = self._initial_weights[0]
               bias_conv = self._initial_weights[1]
               self.b = self._initial_weights[2:]
               self._initial_weights = None
          else:
               self.b = [None]*n
               for i in range(len(self.b)):
                    self.b[i] = self.add_weight(
                                   name="unshift"+str(i),
                                   shape = (self.filters,),
                                   initializer = "zeros",
                                   trainable = True
                                   )

               weights_conv = np.zeros([1, 1, n*self.filters, self.filters])
               for k in range(self.filters):
                    weights_conv[:, :, k::self.filters, k] = 1
               bias_conv = np.zeros(self.filters)

          self.conv = keras.layers.Conv2D(
                    filters=self.filters,
                    kernel_size=1,
                    weights=(weights_conv, bias_conv),
                    **self.kwargs
                    )

          super(NormAdd, self).build(input_shape)

     def call(self, input_data):
          tensor = [None]*len(self.b)
          for i,image in enumerate(input_data):
               tensor[i] = image+self.b[i]

          out = self.concat(tensor)
          out = self.conv(out)

          if self.activation is not None:
               return self.activation(out)
          return out

     def compute_output_shape(self, input_shape):
          return input_shape[0] + (self.filters,)

     def get_config(self):
        config = super().get_config().copy()
        config['weights'] = self.get_weights()
        config.update({
            'activation': self.activation,
            #'filters': self.filters
        })
        return config

     def set_weights(self, weights):
          conv_weights = self.conv.get_weights()
          if len(weights) == len(conv_weights):
               conv_weights = weights
               #print('--',self.name,' - Conv2D weights set.')
          elif len(weights) == len(conv_weights) + len(self.b):
               conv_weights = weights[:len(conv_weights)]
               self.b = [tf.convert_to_tensor(b, dtype=tf.float32) for b in weights[len(conv_weights):]]
               #print('--',self.name,' - Conv2D weights and input biases set.')
          else:
               print('<!!! - ',self.name,'> The weights provided do not match the layer shape. \n \
                    - Conv2D accepts list of len ',len(conv_weights),'. \n \
                    - Input biases accept list of len',len(self.b),'.\n \
                    Provided list of len ', len(weights),':\n        ',[tf.shape(w).numpy().tolist() for w in weights])

          self.conv.set_weights([tf.convert_to_tensor(w, dtype=tf.float32) for w in conv_weights])

     def get_weights(self):
          return self.conv.get_weights()[:2]+self.b

class NormReshape(Layer):

     def __init__(self, target_shape, **kwargs):
          self._initial_weights = kwargs.pop('weights', None)
          self.target_shape = target_shape
          self.resh = keras.layers.Reshape(self.target_shape, **kwargs)
          super(NormReshape, self).__init__(**kwargs)

     def build(self, input_shape):
          self.in_channels = input_shape[-1]

          # Weights
          if self._initial_weights is not None:
               self.lmbda = self._initial_weights[0]
               self.shift = self._initial_weights[1]
               self._initial_weights = None
          else:
               self.lmbda = self.add_weight(
                              name="lambda",
                              shape = (self.in_channels,),
                              initializer = "ones", trainable = False
                              )
               self.shift = self.add_weight(
                              name = "shift",
                              shape = (self.in_channels,),
                              initializer = "zeros", trainable = False
                              )
          super(NormReshape, self).build(input_shape)

     def call(self, input_data):
          out = input_data*(self.lmbda-self.shift)+self.shift
          out = self.resh(out)
          return out

     def get_config(self):
        config = super().get_config().copy()
        config.update({
            'target_shape': self.target_shape,
            'weights': self.get_weights()
        })
        return config

     def get_weights(self):
          return [self.lmbda, self.shift]

     def set_weights(self, weights):
          try:
               self.lmbda = weights[0]
               self.shift = weights[1]
          except ValueError:
               print('Weights need to be of shape: [',tf.shape(self.lmbda),',',tf.shape(self.shift),'].')

class NormConv2D(Conv2D):
     def __init__(self, **kwargs):
          self.initial_weights=kwargs.pop('weights', None)
          super(NormConv2D, self).__init__(**kwargs)

     def build(self, input_shape):
          super(NormConv2D, self).build(input_shape)
          # Normalization
          self.norm = [self.add_weight(
                         name="denom",
                         shape = (input_shape[-1],),
                         initializer = "ones", trainable = False
                         ),
                    self.add_weight(
                         name="shift",
                         shape = (input_shape[-1],),
                         initializer = "zeros", trainable = False
                         )
               ]
          if self.initial_weights is not None:
               self.set_weights(self.initial_weights)
               self._initial_weights = self.initial_weights
               del self.initial_weights

     def call(self, x):
          x = x*self.norm[0] + self.norm[1]
          return Conv2D.call(self, x)

     def set_weights(self,weights):
          if len(weights)==4:
               w = [tf.convert_to_tensor(u) for u in weights]
               super().set_weights(w)
          elif len(weights)==2:
               extra = tf.convert_to_tensor(self.norm)
               super().set_weights(list(weights)+[extra[0]]+[extra[1]])
          else:
               raise ValueError(
               'You called `set_weights(weights)` on layer "%s" '
               'with a weight list of length %s, but the layer was '
               'expecting 4 weights. Provided weights: %s...' %
               (self.name, len(weights), str(weights)[:50]))

     def get_weights(self):
          extra = tf.convert_to_tensor(self.norm)
          return [self.kernel, self.bias, extra[0], extra[1]]

max_stars_repo_name: joaromi/Spiking-RetinaNet

max_issues_repo_name: joaromi/Spiking-RetinaNet

max_forks_repo_name: joaromi/Spiking-RetinaNet


text:
"""
    This program checks whether a hash is observed twice
    or not using bloom filter. For the entries which are repeatative,
    they are logged into a seperate file.

    The analysis is done both on train and test hash.

    Note: If an entry is repeated for N times, it is logged only N-1 times
"""

import os
import sys
import argparse
import xxhash
import json
import math
import prime
import timeit
import numpy as np

class BloomFilter():

    def __init__(self, n, hash_=None,
            seed=None, k=None, m=10**7):
        self.m = m  # Length of bit array
        self.n = n  # Number of elements to store in bloom filter
        self.bit_array = [0] * self.m

        if k:
            self.k = k
        else:
            self.k = self.get_optimal_k()

        self.primes = prime.generate_primes(10000, self.k)
        self.seed = self.get_seed(seed)
        self.hash_func = self.get_hash_func(hash_)

    def get_seed(self, arg):
        if arg == "primes":
            return self.primes
        arg = int(arg)
        seed = [1] * self.k
        seed = [arg * i for i in range(1, self.k + 1)]
        return seed

    def get_hash_func(self, hash_):
        if hash_ == "xxh32":
            return xxhash.xxh32
        elif hash_ == "xxh64":
            return xxhash.xxh64
        elif hash_ == "xxh128":
            return xxhash.xxh128

    def get_optimal_k(self):
        k = int(math.log(2) * self.m / self.n) + 1
        print ("k ", k)
        return  k

    def compute_hash(self, string, seed):
        return self.hash_func(string, seed=seed).intdigest() % self.m

    def add(self, message):
        for i in range(1, self.k+1):
            hash = self.compute_hash(message, seed=self.seed[i-1])
            self.bit_array[hash] = 1

        return

    def check(self, message):
        """
        Computes whether the message is present in the hash function.

        Returns
        -------
            0: if it is not present
            1: if the message is may be present
        """
        for i in range(1, self.k+1):
            hash = self.compute_hash(message, seed=self.seed[i-1])
            if self.bit_array[hash] == 0:
                return 0

        return 1

    def get_fp_probability(self):
        alpha = 1 - math.exp(-1 * self.k * self. n / self.m)
        return alpha ** self.k

if __name__ == "__main__":

    parser = argparse.ArgumentParser()
    parser.add_argument("--hash",
                        help="Hash function to use. One of \
                        xxh32, xxh64, xxh128")
    parser.add_argument("-k", help="Number of hash functions",
                        type=int)
    parser.add_argument("-s", "--seed", help="seed function")
    args = parser.parse_args()

    n = 100000000
    bf = BloomFilter(n, hash_=args.hash, seed=args.seed,
            k=args.k)

    repeat_entries = []
    for file_name in ['logs/train.json', 'logs/test.json']:

        with open(file_name) as f:
            for line in f:
                data = json.loads(line)
                if (data['s_port'] == 443 or data['d_port'] == 443) and data['protocol'] == 6:
                    payload_hash = data['payload_hash']

                    present = bf.check(payload_hash)
                    if present == 0:
                        bf.add(payload_hash)
                    else:
                        repeat_entries.append(line)

    print ("Number of repeat entries ", len(repeat_entries))

    with open('repeats.json', 'w') as f:
        for line in repeat_entries:
            f.write("{}\n".format(line))

max_stars_repo_name: arunppsg/ps-libpcap

max_issues_repo_name: arunppsg/ps-libpcap

max_forks_repo_name: arunppsg/ps-libpcap