text:
# Part of the JBEI Quantitative Metabolic Modeling Library (JQMM)
# Copyright (c) 2016, The Regents of the University of California.
# For licensing details see "license.txt" and "legal.txt".
"""
This module contains all the methods used to treat labeling data (i.e. Mass Distribution Vectors)
"""

import re
import numpy, random
import DB
import utilities as utils

##################################
# ALL TYPES OF LABELING DATA
##################################

# Labeling data classes
class labelingData:
    "General class for storing labeling data data"

    def __init__(self,lines = 'default',dataTup='default',minSTD=0):  # TODO: This init can be improved by using class methods
        if lines !='default':
            # Unpacking lines input = tuple of MS line and MS stds line
            if lines.__class__.__name__ == 'tuple':
                line,STDline = lines
            else:
                line    = lines
                STDline = ''
            # Eliminate header from lines
            line      = line.split(':')[-1]

            if STDline:
                STDline = STDline.split(':')[-1]
            # Components from line
            self.fragment = self.getFragmentName(line)
            self.inFit = True   # This decides wether fragment info is included in the fit
            self.abbrev   = self.getAbbrev(self.fragment)

            comps    = line.split('\t')

            # MDV (and STD if present) values
            values = '\t '.join(comps[2:])
            MDVSTDs = re.findall('([\.\d\-eE]+)\(([\.\d\-eE]+)\)',values)      # input can be in the form mdv(std): 0.234(0.030) or the form mdv: 0.234
            MDVs    = re.findall('([\.\d\-eE]+)',values)

            MDVlist = []
            STDlist = []
            if MDVSTDs:
                for tup in MDVSTDs:
                    mdv, STD = tup
                    #mdv = 0 if mdv == '-' else mdv
                    if not mdv == '-':
                        MDVlist.append(float(mdv))
                        STDlist.append(float(STD))
            else:
                if MDVs:
                    for mdv in MDVs:
                    #    mdv = 0 if mdv == '-' else mdv
                        if not mdv == '-':
                            MDVlist.append(float(mdv))
                else:
                    raise Exception('Line input cannot be parsed: '+line)

            # STD values
            if STDline:
                STDfragment = self.getAbbrev(STDline.split('\t')[0].strip())
                if STDfragment != self.fragment:
                    raise Exception('STD line fragment name does not correspond to MDV line fragment: '+STDfragment+' vs '+self.fragment)
                else:
                    STDs    = re.findall('([\.\d]+)','\t'.join(STDline.split('\t')[2:]))
                    STDlist = []
                    for STD in STDs:
                        STDlist.append(float(STD))
            else:
                if not STDlist:                    # Fill up the standard deviation with default result if nothing else is available
                    STDlist = list(0.03*numpy.ones(shape=(len(MDVlist))))
            # Capping minimum value of STDs
            STDlistMod = []
            for value in STDlist:
                value = max(value,minSTD)
                STDlistMod.append(value)

            self.mdv = numpy.array(MDVlist)

            STDlist = STDlist[0:len(MDVlist)]
            self.std = numpy.array(STDlistMod)

        elif dataTup !='default':                    # This part of labeling data initialization should be updated
            fragment, mdv, STD = dataTup

            self.fragment = fragment
            self.mdv = mdv
            if STD:
                self.std = STD
            else:
                self.std = numpy.array(list(0.03*numpy.ones(shape=(len(self.mdv)))))
            self.abbrev = self.getAbbrev(fragment)

        else:
            raise Exception('Incorrect input: must input lines or MDV info')

        self.line = self.getLine()

    def getMval(self,fragment):
        [aa,mVal] = fragment.split('M-')
        return mVal

    def getLine(self,maxVals=13):
        "Converts MDVs and STDs into a single string line"
        base     = '-\t'
        fragment = self.fragment
        mdv      = self.mdv
        try:
            std    = self.std
            stdStr = []
            for i in range(len(std)):
                stdStr.append('{0:2.3f}'.format(std[i]))
            for i in range(len(std),len(mdv)):
                stdStr.append('0')
        except AttributeError:
            # No std provided
            for i in range(len(mdv)):
                stdStr.append('--')

        MDVlist = []

        for i in range(len(mdv)):
            MDVlist.append('{0:2.3f}'.format(mdv[i])+'('+stdStr[i]+')')
        mVal = self.getMval(fragment)
        name = self.getName()               # This should probably be done through the fragment class
        line = name+'\t'+'M-'+mVal+'\t'+'\t'.join(MDVlist)+'\t'+base*(maxVals-len(MDVlist))

        return line

    def randomize(self):
        mdv    = self.mdv
        STD    = self.std
        newMDV = []
        for i in range(len(mdv)):
            val = mdv[i]
            std = STD[i]
            newVal = numpy.maximum(val+(2*random.random()-1)*std,0)
            newMDV.append(newVal)
        self.mdv = numpy.array(newMDV)

        # Update line
        self.line = self.getLine()

    def normalize(self):
        total = sum(self.mdv)
        if total != 0:
            self.mdv   = self.mdv / total

        # Update line
        self.line = self.getLine()

class LCMSLabelData(labelingData):
    "Class for storing LCMS data"

    def getName(self):
        return self.abbrev

    def getMval(self,fragment):
        return '0'

    def getAbbrev(self,fragment,standard = False):
        "Creates abbreviation for fragment"
        abbrev = fragment.strip()      # e.g. 3pg

        if standard:     # Generate standarized alias
            abbrev, warning = getStdAlias(abbrev)
            if not abbrev:
                print warning

        return abbrev

    def getFragmentName(self,line):
        "Creates abbreviation for fragment"
        comps  = line.split('\t')

        part1 = comps[0].strip()
        part2 = comps[1].strip().replace('M-0','',1).replace('M-','',1).strip()

        fragmentName = part1+part2

        return fragmentName

class CEMSLabelData(LCMSLabelData):
    "Class for storing CEMS data"
    def nothing(self):
        "I can't have an inherited class without adding something, it seems."
        pass

class GCMSLabelData(labelingData):
    "Class for storing GCMS data"

    def getAbbrev(self,fragment,standard = False):
        "Creates abbreviation for fragment"
        abbrev = fragment.replace('M-','',1)      # e.g. Ala159
        number = re.search('(\d+)',abbrev).group(1)
        abbrev = abbrev.replace(number,'',1)

        if standard:     # Generate standarized alias
            abbrev, warning = getStdAlias(abbrev)
            if not abbrev:
                print warning

        return abbrev+number

    def getName(self):
        name = re.sub('(\d+)','',self.abbrev)                # This should probably be done through the fragment class
        return name

    def getFragmentName(self,line):
        "Creates abbreviation for fragment"
        comps  = line.split('\t')
        fragmentName = comps[0].strip()+comps[1]
        return fragmentName

    def getMval(self,fragment):
        [aa,mVal] = fragment.split('M-')
        return mVal

# Classes for fragments (GCMS,LCMS....)

class fragment():
    """ A base class for all fragments"""

    def __init__(self,emu,name,abbrev,ncarbons,composition):
        """
        emu: emu name (e.g. alaL_2_3)
        name: fragment name (e.g. Alanine,159)
        abbrev: fragment abbreviation (e.g. Ala159)
        ncarbons: number of carbons (e.g. 2)
        composition: fragment composition minus carbon backbone (e.g. C6H20NSi)
        """

        self.emu      = emu
        self.name     = name
        self.abbrev   = abbrev
        self.ncarbons = ncarbons
        self.comp     = composition

    def __str__(self):
        string = self.name
        if hasattr(self, 'mdv'):
            string = string + ': ' + str(self.mdv)
        if hasattr(self, 'std'):
            string = string + ': ' + str(self.std)
        return string

class GCMSfragment(fragment):
    """ A class for GCMS fragments"""

    def __init__(self,emu,name,abbrev,ncarbons,composition,antonApproved,derWeight,derAtoms):
        """
        emu: emu name (e.g. alaL_2_3)
        name: fragment name (e.g. Alanine,159)
        abbrev: fragment abbreviation (e.g. Ala159)
        ncarbons: number of carbons (e.g. 2)
        composition: fragment composition minus carbon backbone (e.g. C6H20NSi)
        antonApproved: approved by antoniewicz
        derWeight: weight of derivatized fragment
        derAtoms: composition of derivatized fragment
        """

        fragment.__init__(self,emu,name,abbrev,ncarbons,composition)

        self.antonAp  = antonApproved
        self.derWeigth= derWeight
        self.derAtoms = derAtoms

class LCMSfragment(fragment):
    """ A class for LCMS fragments"""

    def nothing(self):
        "I can't have an inherited class without adding something, it seems."
        pass

class CEMSfragment(fragment):
    """ A class for CEMS fragments"""

    def nothing(self):
        "I can't have an inherited class without adding something, it seems."
        pass

class LabelinglString():
    """ An class for strings containing labeling info
        Example input:
                 lstring: U
                 ncarbons: 6
    """
    # Adding data input as part of the init
    def __init__(self, lstring, ncarbons):
        if lstring == 'U':
            # Fully labeled
            self.labelCarbons = ''.zfill(ncarbons).replace('0','1')
        elif lstring == '1-C':
            # First Carbon labeled
            self.labelCarbons = '1'+''.zfill(ncarbons-1)
        elif lstring == '2-C':
            # Second Carbon labeled
            self.labelCarbons = '01'+''.zfill(ncarbons-2)
        elif lstring == '1,2-C':
            # First 2 carbons labeled
            self.labelCarbons = '11'+''.zfill(ncarbons-2)
        elif lstring == '2,3-C':
            # Second and third carbons labeled
            self.labelCarbons = '011'+''.zfill(ncarbons-3)
        elif lstring == 'UN':
            # Unlabeled
            self.labelCarbons = ''.zfill(ncarbons)
        else:
            raise Exception('Feed labeling code not recognized: ' + str(lstring))
        if self.labelCarbons == '':
            raise Exception('Labeling string %s with ncarbons %s resulted in empty self.labelCarbons' % (lstring, str(ncarbons)))

    def mdv(self, emu):
        # Find list of atom positions involved in emu.

        # This turns a string like 'glc_D_e_3_4_5_6' into an array ['3', '4', '5', '6']
        elList = re.search('((_\d)+)', emu).group(0).split('_')[1:]
        mval = 0
        for el in elList:
            try:
                mval = mval + int(self.labelCarbons[int(el)-1])
            except IndexError:
                raise Exception('Trying to address index %s in self.labelCarbons "%s", based on emu "%s"' % (int(el)-1, self.labelCarbons, emu))
        mdv = []
        for i in range(len(elList)+1):
            mdv.append(0)
        mdv[mval] = 1
        return numpy.array(mdv)

########
# Auxiliary functions
########

def getStdAlias(name):
    """
    Generates standard alias for a metabolite name, as from fragdictbasic

    For example:

    getStdAlias('Asp-L') = ('Asp,'')
    """

    aliases        = genAliases(name)
    fragDictBasic_ = DB.fragDictBasic()
    presenceVector = [i in fragDictBasic_ for i in aliases]  # Vector with presence or absence for each alias
    if sum(presenceVector)>0:                                # checks if any of the aliases is in fragDictBasic keys
        newName = aliases[presenceVector.index(True)]        # Abbreviation name present in fragDictBasic
        warning = ''
    else:
        newName = ''
        warning = 'Warning: fragment '+str(name)+' not found in fragDictBasic:\n'+str(fragDictBasic_.keys())

    return newName, warning

# Function which generates aliases to metabolite names
def genAliases(name):
    """
    Generates aliases for metabolite names, e.g.:

        val --> set(['Val-L', 'Val', 'val', 'val-L'])

    """
    name = name.replace('-L','').replace('-l','')
    output = []
    output.append(name)
    output.append(name.lower())
    output.append(name.lower()+'-L')
    output.append(name.lower()+'_L')
    output.append(name.capitalize())
    output.append(name.capitalize()+'-L')
    output.append(name.capitalize()+'_L')

    return output

def createGammaMatrix(distleng,string): # TODO: find a better place for this
    """
    Function to create gamma matrix for the atoms found in the input string as per Wahl et al Biotechnol Bioeng 85: 259-268 (2004)
        -) distleng is the length of the distribution the matrix has to be multiplied by.
        Example: gammarray(10,'C3H2O')
    """
    # Natural isotope abundances
    abundDict = {
        'C' :numpy.array([[12,0.98918],[13,0.01082]]),
        'O' :numpy.array([[16,0.99756],[17,0.00038],[18,0.00206]]),
        'H' :numpy.array([[1,0.999844],[2,0.000156]]),
        'N' :numpy.array([[14,0.99634],[15,0.00366]]),
        'Si':numpy.array([[28,0.9222],[29,0.0469],[30,0.0309]]),
        'S' :numpy.array([[32,0.95039],[33,0.00749],[34,0.04197],[36,0.00015]]),
        'P' :numpy.array([[31,1]])
        }

    # Getting atom species and number
    atoms    = utils.getAtoms(string)

    # Initialization
    gammatot = numpy.eye(distleng)

    # Main loop
    for atom in atoms:
        symbol, number = atom
        if symbol in abundDict:
            for j in range(number):
                # Make gamma matrix
                distribution = abundDict[symbol][:,1]
                gamma = numpy.zeros((distleng+len(distribution)-1,distleng))
                for j in range(distleng):
                    gamma[j:j+len(distribution),j] = distribution.transpose()
                # Add to total gamma
                gammatot = numpy.matrix(gamma)*gammatot
                distleng = gamma.shape[0]
        else:
            raise Exception ('Atom '+symbol+' not in table, please add')

    return gammatot

max_stars_repo_name: GBirkel/jqmm

max_issues_repo_name: zhwycsz/jqmm

max_forks_repo_name: zhwycsz/jqmm


text:
import numpy as np
import scipy.sparse as sp
from sklearn.feature_extraction import DictVectorizer
from splearn.feature_extraction import SparkDictVectorizer
from splearn.rdd import ArrayRDD
from splearn.utils.testing import (SplearnTestCase, assert_array_equal,
                                   assert_equal, assert_true)
from splearn.utils.validation import check_rdd_dtype

class TestDictVectorizer(SplearnTestCase):

    def make_dict_dataset(self, blocks=-1):
        X = [{"foo": 1, "bar": 3},
             {"bar": 4, "baz": 2},
             {"bar": 6, "baz": 1},
             {"bar": 4, "ewo": "ok"},
             {"bar": 4, "baz": 2},
             {"bar": 9, "ewo": "fail"},
             {"bar": 4, "baz": 2},
             {"bar": 1, "quux": 1, "quuux": 2}]
        X_rdd = ArrayRDD(self.sc.parallelize(X, 4), blocks)
        return X, X_rdd

    def test_same_output_dense(self):
        X, X_rdd = self.make_dict_dataset()
        local = DictVectorizer(sparse=False)
        dist = SparkDictVectorizer(sparse=False)

        result_local = local.fit_transform(X)
        result_dist = dist.fit_transform(X_rdd)

        assert_true(check_rdd_dtype(result_dist, (np.ndarray,)))
        assert_equal(local.vocabulary_, dist.vocabulary_)
        assert_array_equal(result_local, result_dist.toarray())

    def test_same_output_sparse(self):
        X, X_rdd = self.make_dict_dataset()
        local = DictVectorizer(sparse=True)
        dist = SparkDictVectorizer(sparse=True)

        result_local = local.fit_transform(X)
        result_dist = dist.fit_transform(X_rdd)

        assert_true(check_rdd_dtype(result_dist, (sp.spmatrix,)))
        assert_equal(local.vocabulary_, dist.vocabulary_)
        assert_array_equal(result_local.toarray(), result_dist.toarray())

ext: py

max_stars_repo_name: dtrckd/sparkit-learn

max_issues_repo_name: dtrckd/sparkit-learn

max_forks_repo_name: dtrckd/sparkit-learn

reason: import numpy,import scipy


text:
"""
Unit tests for the 2D electromagnetic PIC that may not be deterministic
and tests a graph output. Different quantities are computed and graphed,
and then the test asks for user input regarding whether the graphs are
correct.
"""

import unittest
import py_platypus as plat
import numpy as np
import matplotlib.pyplot as plt
import unit_helpers

class Pic2dEmGraphTester(unittest.TestCase):
    """
    Unit tests for electromagnetic 2D PIC that generates graphs.
    """

    def setUp(self):
        """
        set up function called before each test
        """

        self.pic = unit_helpers.setup("2d_em")

    def test_random_x(self):
        """
        Tests initial random distribution of particles.
        """
        print("==== Testing random x ====")
        self.pic.init_x_random()
        plt.figure(1)

        plt.scatter(self.pic.electron_x, self.pic.electron_y, s=0.05)
        expect_str = "\nRandom distribution of points"

        self.assertEqual(unit_helpers.check_graph(expect_str), True, "random x image does not match expected")

        return

    def test_init_e(self):
        '''
        Tests calculating the inital electric field, both at the cell centers
        at initialization (from a Poisson solver) and interpolated to the
        edges.
        '''
        print("==== Test init E ====")
        # create an initial density perturbation and calculate the electric field
        self.pic.init_x_random()
        self.pic.init_v_maxwellian()
        self.pic.density_perturbation()
        self.pic.init_E()

        # Ex electric field should be sinusoidal
        plt.figure(1)
        plt.imshow(self.pic.ex)
        plt.title("Ex field")
        plt.colorbar()

        # Ey electric field should be random
        plt.figure(2)
        plt.imshow(self.pic.ey)
        plt.title("Ey field")
        plt.colorbar()

        # Ex electric field should be sinusoidal
        plt.figure(3)
        plt.imshow(self.pic.ex_edges)
        plt.title("Ex field edges")
        plt.colorbar()

        # Ey electric field should be random
        plt.figure(4)
        plt.imshow(self.pic.ey_edges)
        plt.title("Ey field edges")
        plt.colorbar()

        expect_str = "\nEx and Ex field edges sinusoidal and similar; Ey and Ey fields edges random and similar"

        self.assertEqual(unit_helpers.check_graph(expect_str), True, "init E field image does not match expected")

        return

    def test_update_b(self):
        '''
        Tests calculating the magnetic field update based on the electric
        field.
        '''

        print("==== Test update B ====")
        # set up x vectors
        rows, cols = self.pic.ex_edges.shape
        for i in range(rows):
            for j in range(cols):
                x = i - rows/2
                y = j - cols/2

                self.pic.ex_edges[i][j] = y

        # set up y vectors
        rows, cols = self.pic.ey_edges.shape
        for i in range(rows):
            for j in range(cols):
                x = i - rows/2
                y = j - cols/2

                self.pic.ey_edges[i][j] = -x

        self.pic.calc_B_update()
        plt.figure(1)
        plt.imshow(self.pic.delta_bz)
        plt.title("B update for uniform curl")
        expect_str = "\nUniform graph"

        self.assertEqual(unit_helpers.check_graph(expect_str), True, "init B field image does not match expected")

if __name__ == "__main__":
    unittest.main()

max_stars_repo_name: timsliu/platypus

max_issues_repo_name: timsliu/platypus

max_forks_repo_name: timsliu/platypus

reason: import numpy


text:
#!/usr/bin/env python3
'''
 * @file fcn8_inference.py
 * @brief Used for training neural nets, validating the nets by viewing the predictions, viewing the images passed into the network after preprocessing, and deployment
 *
 * @author Matthew Powelson
 * @date November 6, 2019
 * @version TODO
 * @bug No known bugs
 *
 * @copyright Copyright (c) 2017, Southwest Research Institute
 *
 * @par License
 * Software License Agreement (Apache License)
 * @par
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * http://www.apache.org/licenses/LICENSE-2.0
 * @par
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 '''

# For data loading and visualization
import numpy as np
import cv2 as cv

# For importing the config
import json
import os

# Inference module
from pcs_detection.inference import Inference

# Hack because code was originally written to have configs as Python modules
class Config:
    def __init__(self, **entries):
        self.__dict__.update(entries)

if __name__ == '__main__':
  # Import Config json file and convert into format we need
  dir_path = os.path.dirname(os.path.realpath(__file__))

  with open(dir_path + '/data/weights/<weight_path>/inference_config.json') as json_data_file:
      data = json.load(json_data_file)
  config = Config(**data)

  # Construct the annotator
  annotator = Inference(config)

  # Load the image
  input_image = cv.imread('<image you want to test>')
  # Generate the annotation and convert to 3 channel image
  res = annotator.make_prediction(input_image)

  display_image =  np.zeros((res.shape[0], res.shape[1], 3))
  colors = [(102, 204, 255), (255, 153, 102), (0,153,255)]
  for jj, color in enumerate(colors):
        temp_chnl_img = np.zeros((res.shape[0],res.shape[1], 2))
        temp_chnl_img[:,:,1][res==jj+1] = 1
        display_image[temp_chnl_img[:,:,1]==1] = color
  display_image = display_image.astype(np.uint8)

  # Show the results
  print("Input image of size: " + str(input_image.shape))
  print("Results image of size: " + str(display_image.shape))
  print("Press ESC to exit")
  while True:
      cv.imshow("image", np.hstack((input_image, display_image)))
      k = cv.waitKey(1) & 0xFF
      if k == 27:
          cv.destroyWindow("image")
          break


text:
""" Variational Auto-Encoder Example.

Using a variational auto-encoder to generate digits images from noise.
MNIST handwritten digits are used as training examples.

References:
    - Auto-Encoding Variational Bayes The International Conference on Learning
    Representations (ICLR), Banff, 2014. D.P. Kingma, M. Welling
    - Understanding the difficulty of training deep feedforward neural networks.
    X Glorot, Y Bengio. Aistats 9, 249-256
    - Y. LeCun, L. Bottou, Y. Bengio, and P. Haffner. "Gradient-based
    learning applied to document recognition." Proceedings of the IEEE,
    86(11):2278-2324, November 1998.

Links:
    - [VAE Paper] https://arxiv.org/abs/1312.6114
    - [Xavier Glorot Init](www.cs.cmu.edu/~bhiksha/courses/deeplearning/Fall.../AISTATS2010_Glorot.pdf).
    - [MNIST Dataset] http://yann.lecun.com/exdb/mnist/

Author: Aymeric Damien
Project: https://github.com/aymericdamien/TensorFlow-Examples/
"""
from __future__ import division, print_function, absolute_import

import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import norm
import tensorflow as tf

# Import MNIST data
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)

# Parameters
learning_rate = 0.001
num_steps = 30000
batch_size = 64

# Network Parameters
image_dim = 784 # MNIST images are 28x28 pixels
hidden_dim = 512
latent_dim = 2

# A custom initialization (see Xavier Glorot init)
def glorot_init(shape):
    return tf.random_normal(shape=shape, stddev=1. / tf.sqrt(shape[0] / 2.))

# Variables
weights = {
    'encoder_h1': tf.Variable(glorot_init([image_dim, hidden_dim])),
    'z_mean': tf.Variable(glorot_init([hidden_dim, latent_dim])),
    'z_std': tf.Variable(glorot_init([hidden_dim, latent_dim])),
    'decoder_h1': tf.Variable(glorot_init([latent_dim, hidden_dim])),
    'decoder_out': tf.Variable(glorot_init([hidden_dim, image_dim]))
}
biases = {
    'encoder_b1': tf.Variable(glorot_init([hidden_dim])),
    'z_mean': tf.Variable(glorot_init([latent_dim])),
    'z_std': tf.Variable(glorot_init([latent_dim])),
    'decoder_b1': tf.Variable(glorot_init([hidden_dim])),
    'decoder_out': tf.Variable(glorot_init([image_dim]))
}

# Building the encoder
input_image = tf.placeholder(tf.float32, shape=[None, image_dim])
encoder = tf.matmul(input_image, weights['encoder_h1']) + biases['encoder_b1']
encoder = tf.nn.tanh(encoder)
z_mean = tf.matmul(encoder, weights['z_mean']) + biases['z_mean']
z_std = tf.matmul(encoder, weights['z_std']) + biases['z_std']

# Sampler: Normal (gaussian) random distribution
eps = tf.random_normal(tf.shape(z_std), dtype=tf.float32, mean=0., stddev=1.0,
                       name='epsilon')
z = z_mean + tf.exp(z_std / 2) * eps

# Building the decoder (with scope to re-use these layers later)
decoder = tf.matmul(z, weights['decoder_h1']) + biases['decoder_b1']
decoder = tf.nn.tanh(decoder)
decoder = tf.matmul(decoder, weights['decoder_out']) + biases['decoder_out']
decoder = tf.nn.sigmoid(decoder)

# Define VAE Loss
def vae_loss(x_reconstructed, x_true):
    # Reconstruction loss
    encode_decode_loss = x_true * tf.log(1e-10 + x_reconstructed) \
                         + (1 - x_true) * tf.log(1e-10 + 1 - x_reconstructed)
    encode_decode_loss = -tf.reduce_sum(encode_decode_loss, 1)
    # KL Divergence loss
    kl_div_loss = 1 + z_std - tf.square(z_mean) - tf.exp(z_std)
    kl_div_loss = -0.5 * tf.reduce_sum(kl_div_loss, 1)
    return tf.reduce_mean(encode_decode_loss + kl_div_loss)

loss_op = vae_loss(decoder, input_image)
optimizer = tf.train.RMSPropOptimizer(learning_rate=learning_rate)
train_op = optimizer.minimize(loss_op)

# Initialize the variables (i.e. assign their default value)
init = tf.global_variables_initializer()

# Start training
with tf.Session() as sess:

    # Run the initializer
    sess.run(init)

    for i in range(1, num_steps+1):
        # Prepare Data
        # Get the next batch of MNIST data (only images are needed, not labels)
        batch_x, _ = mnist.train.next_batch(batch_size)

        # Train
        feed_dict = {input_image: batch_x}
        _, l = sess.run([train_op, loss_op], feed_dict=feed_dict)
        if i % 1000 == 0 or i == 1:
            print('Step %i, Loss: %f' % (i, l))

    # Testing
    # Generator takes noise as input
    noise_input = tf.placeholder(tf.float32, shape=[None, latent_dim])
    # Rebuild the decoder to create image from noise
    decoder = tf.matmul(noise_input, weights['decoder_h1']) + biases['decoder_b1']
    decoder = tf.nn.tanh(decoder)
    decoder = tf.matmul(decoder, weights['decoder_out']) + biases['decoder_out']
    decoder = tf.nn.sigmoid(decoder)

    # Building a manifold of generated digits
    n = 20
    x_axis = np.linspace(-3, 3, n)
    y_axis = np.linspace(-3, 3, n)

    canvas = np.empty((28 * n, 28 * n))
    for i, yi in enumerate(x_axis):
        for j, xi in enumerate(y_axis):
            z_mu = np.array([[xi, yi]] * batch_size)
            x_mean = sess.run(decoder, feed_dict={noise_input: z_mu})
            canvas[(n - i - 1) * 28:(n - i) * 28, j * 28:(j + 1) * 28] = \
            x_mean[0].reshape(28, 28)

    plt.figure(figsize=(8, 10))
    Xi, Yi = np.meshgrid(x_axis, y_axis)
    plt.imshow(canvas, origin="upper", cmap="gray")
    plt.show()

max_stars_repo_name: kar-thik/TensorFlow-Examples

max_issues_repo_name: kar-thik/TensorFlow-Examples

max_forks_repo_name: kar-thik/TensorFlow-Examples

reason: import numpy,from scipy


text:
from visual_mpc.envs.robot_envs.base_env import BaseRobotEnv
import copy
from visual_mpc.envs.util.action_util import autograsp_dynamics
import numpy as np
import logging

class AutograspEnv(BaseRobotEnv):
    def __init__(self, env_params, _=None):
        assert 'mode_rel' not in env_params, "Autograsp sets mode_rel"

        self._hyper = copy.deepcopy(env_params)

        BaseRobotEnv.__init__(self, self._hyper)
        self._adim, self._sdim = 4, self._base_sdim

    def _init_dynamics(self):
        self._gripper_closed = False
        self._prev_touch = False

    def _default_hparams(self):
        default_dict = {'zthresh': 0.15,
                        'gripper_joint_grasp_min': 0.0,
                        'gripper_joint_thresh': -1.,   # anything <0 deactivates this check
                        'reopen': True,
                        'robot_upside_down': False}

        parent_params = BaseRobotEnv._default_hparams(self)
        for k in default_dict.keys():
            parent_params.add_hparam(k, default_dict[k])
        return parent_params

    def _next_qpos(self, action):
        assert action.shape[0] == 4      # z dimensions are normalized across robots
        norm_gripper_z = (self._previous_target_qpos[2] - self._low_bound[2]) / \
                         (self._high_bound[2] - self._low_bound[2])
        if self._hp.robot_upside_down:
            norm_gripper_z = 1 - norm_gripper_z

        z_thresh = self._hp.zthresh

        joint_test = self._last_obs['state'][-1] > self._hp.gripper_joint_grasp_min and \
                     np.abs(self._last_obs['state'][-1]) < self._hp.gripper_joint_thresh
        touch_test = joint_test or np.amax(self._last_obs.get('finger_sensors', 0)) > 0
        logging.getLogger('robot_logger').debug('joint is {} and test is {}'.format(self._last_obs['state'][-1], joint_test))

        target, self._gripper_closed = autograsp_dynamics(self._previous_target_qpos, action,
                                                          self._gripper_closed, norm_gripper_z, z_thresh,
                                                          self._hp.reopen, touch_test or self._prev_touch,
                                                          self._low_bound[-1], self._high_bound[-1])
        logging.getLogger('robot_logger').debug('norm_gripper_z is {} and close command is {}'.format(norm_gripper_z, self._gripper_closed))
        self._prev_touch = touch_test
        return target


text:
#!/usr/bin/env python3

"""
    # {student full name}
    # {student id}
    # {student email}
"""

# Python standard library
from math import cos, sin, atan2, fabs

# Numpy
import numpy as np

# "Local version" of ROS messages
from local.geometry_msgs import PoseStamped, Quaternion
from local.sensor_msgs import LaserScan
from local.map_msgs import OccupancyGridUpdate

from grid_map import GridMap

class Mapping:
    def __init__(self, unknown_space, free_space, c_space, occupied_space,
                 radius, optional=None):
        self.unknown_space = unknown_space
        self.free_space = free_space
        self.c_space = c_space
        self.occupied_space = occupied_space
        self.allowed_values_in_map = {"self.unknown_space": self.unknown_space,
                                      "self.free_space": self.free_space,
                                      "self.c_space": self.c_space,
                                      "self.occupied_space": self.occupied_space}
        self.radius = radius
        self.__optional = optional

    def get_yaw(self, q):
        """Returns the Euler yaw from a quaternion.
        :type q: Quaternion
        """
        return atan2(2 * (q.w * q.z + q.x * q.y),
                     1 - 2 * (q.y * q.y + q.z * q.z))

    def raytrace(self, start, end):
        """Returns all cells in the grid map that has been traversed
        from start to end, including start and excluding end.
        start = (x, y) grid map index
        end = (x, y) grid map index
        """
        (start_x, start_y) = start
        (end_x, end_y) = end
        x = start_x
        y = start_y
        (dx, dy) = (fabs(end_x - start_x), fabs(end_y - start_y))
        n = dx + dy
        x_inc = 1
        if end_x <= start_x:
            x_inc = -1
        y_inc = 1
        if end_y <= start_y:
            y_inc = -1
        error = dx - dy
        dx *= 2
        dy *= 2

        traversed = []
        for i in range(0, int(n)):
            traversed.append((int(x), int(y)))

            if error > 0:
                x += x_inc
                error -= dy
            else:
                if error == 0:
                    traversed.append((int(x + x_inc), int(y)))
                y += y_inc
                error += dx

        return traversed

    def add_to_map(self, grid_map, x, y, value):
        """Adds value to index (x, y) in grid_map if index is in bounds.
        Returns weather (x, y) is inside grid_map or not.
        """
        if value not in self.allowed_values_in_map.values():
            raise Exception("{0} is not an allowed value to be added to the map. "
                            .format(value) + "Allowed values are: {0}. "
                            .format(self.allowed_values_in_map.keys()) +
                            "Which can be found in the '__init__' function.")

        if self.is_in_bounds(grid_map, x, y):
            grid_map[x, y] = value
            return True
        return False

    def is_in_bounds(self, grid_map, x, y):
        """Returns weather (x, y) is inside grid_map or not."""
        if x >= 0 and x < grid_map.get_width():
            if y >= 0 and y < grid_map.get_height():
                return True
        return False

    def update_map(self, grid_map, pose, scan):
        """Updates the grid_map with the data from the laser scan and the pose.

        For E:
            Update the grid_map with self.occupied_space.

            Return the updated grid_map.

            You should use:
                self.occupied_space  # For occupied space

                You can use the function add_to_map to be sure that you add
                values correctly to the map.

                You can use the function is_in_bounds to check if a coordinate
                is inside the map.

        For C:
            Update the grid_map with self.occupied_space and self.free_space. Use
            the raytracing function found in this file to calculate free space.

            You should also fill in the update (OccupancyGridUpdate()) found at
            the bottom of this function. It should contain only the rectangle area
            of the grid_map which has been updated.

            Return both the updated grid_map and the update.

            You should use:
                self.occupied_space  # For occupied space
                self.free_space      # For free space

                To calculate the free space you should use the raytracing function
                found in this file.

                You can use the function add_to_map to be sure that you add
                values correctly to the map.

                You can use the function is_in_bounds to check if a coordinate
                is inside the map.

        :type grid_map: GridMap
        :type pose: PoseStamped
        :type scan: LaserScan
        """

        # Current yaw of the robot
        robot_yaw = self.get_yaw(pose.pose.orientation)
        # The origin of the map [m, m, rad]. This is the real-world pose of the
        # cell (0,0) in the map.
        origin = grid_map.get_origin()
        # The map resolution [m/cell]
        resolution = grid_map.get_resolution()

        """
        Fill in your solution here
        """

        """
        For C only!
        Fill in the update correctly below.
        """
        # Only get the part that has been updated
        update = OccupancyGridUpdate()
        # The minimum x index in 'grid_map' that has been updated
        update.x = 0
        # The minimum y index in 'grid_map' that has been updated
        update.y = 0
        # Maximum x index - minimum x index + 1
        update.width = 0
        # Maximum y index - minimum y index + 1
        update.height = 0
        # The map data inside the rectangle, in row-major order.
        update.data = []

        # Return the updated map together with only the
        # part of the map that has been updated
        return grid_map, update

    def inflate_map(self, grid_map):
        """For C only!
        Inflate the map with self.c_space assuming the robot
        has a radius of self.radius.

        Returns the inflated grid_map.

        Inflating the grid_map means that for each self.occupied_space
        you calculate and fill in self.c_space. Make sure to not overwrite
        something that you do not want to.

        You should use:
            self.c_space  # For C space (inflated space).
            self.radius   # To know how much to inflate.

            You can use the function add_to_map to be sure that you add
            values correctly to the map.

            You can use the function is_in_bounds to check if a coordinate
            is inside the map.

        :type grid_map: GridMap
        """

        """
        Fill in your solution here
        """

        # Return the inflated map
        return grid_map


text:
from jaxns.prior_transforms.common import *
from jaxns.prior_transforms.deterministic import *
from jaxns.prior_transforms.identifiable import *
from jaxns.prior_transforms.levy import *
from jaxns.prior_transforms.mixture import *
from jaxns.prior_transforms.prior_chain import PriorChain

max_stars_repo_name: fehiepsi/jaxns

max_issues_repo_name: fehiepsi/jaxns

max_forks_repo_name: fehiepsi/jaxns

reason: from jax


text:
"""
mean_shift_data_preparation.py

Functions to be called in order that run machine learning: mean shift
clustering algorithm.  The algorithm uses common words as feature words
(presence of the word is converted into a boolean for the feature list).
Bandwith can be flexed to adjust number of clusters and number of profiles
per cluster.

Run functions that are commented out under the __name__ == "__main__"
as appropriate.
"""

import os

parentdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
os.sys.path.insert(0,parentdir)

from model import Profile, Adjective, Gender, Orientation, Location, db, connect_to_db, MeanShiftAlgo, SelfSummaryLabel, MessageMeIfLabel
from flask_app import app
from sklearn.cluster import MeanShift, estimate_bandwidth
from random import sample
import numpy as np
import datetime
from sklearn.externals import joblib
from nltk.corpus import stopwords

def clean(dirty_list):
    """ Returns list of words with non-alpha characters stripped """

    clean_list = []
    clean_string = ""
    for word in dirty_list:
        word = unicode(word)
        word = word.strip("\n\t1234567890!@#$%^&*()-=+{}[]/\\'\"<>?.,:;~`").replace("\n", " ").replace(".", " ").replace("\t","").replace("1","").replace("2","").replace("3","").replace("4","").replace("5","").replace("6","").replace("7","").replace("8","").replace("9","").replace("0","").replace("!","").replace("@","").replace("#","").replace("$","").replace("%","").replace("^","").replace("&","").replace("*","").replace("(","").replace(")","").replace("-"," ").replace("=","").replace("+","").replace("{","").replace("}","").replace("[","").replace("]","").replace("'","").replace("\"","").replace(":","").replace(";","").replace("\\","").replace("/","").replace(">","").replace("<","").replace("`","").replace("~","").replace("_","")
        clean_string += word+" "

    clean_list.extend(clean_string.split(" "))

    return clean_list

def get_words(profile, random_profiles_to_train, n_train):
    """Gets word (by parsing text) out of database based on random samples.

    Remove from list common words that appear above threshold (n) times.
    Returns a randomly generated list of features.
    """
    low_threshold, high_threshold = n_train*0.05, n_train*.5

    text_tuples = db.session.query(profile).filter(Profile.username.in_(random_profiles_to_train)).all()

    words_and_count = {}

    features_set = set()

    text_word_list = []
    for text in text_tuples:
        text_word_list.extend(text[0].split(" "))

    clean_list = clean(text_word_list)

    filtered_words = [word for word in clean_list if word not in stopwords.words('english')]

    for word in filtered_words:

        words_and_count[word] = words_and_count.get(word,0)
        words_and_count[word] += 1

    for key, value in words_and_count.iteritems():
        if value <= high_threshold and value >= low_threshold:
            features_set.add(key)

    features = list(features_set)

    return features

def convert(features, text_list):
    """For each user, search features, and return numpy boolean array for each user

    1 if has feature, 0 if doesn't
    """

    boolean_input = []

    for text in text_list:

        user = []
        clean_list = clean(text.split(" "))
        for feature in features:
            word_count = max([1 if word == feature else 0 for word in clean_list])
            user.append(word_count)

        boolean_input.append(user)

    boolean_input_array = np.array(boolean_input, dtype=int)

    shape = boolean_input_array.shape

    return boolean_input_array

def mean_shift_train(boolean_input_array, bandwidth_factor):
    """Conduct mean shift analysis on training data."""

    n_samples, n_features = boolean_input_array.shape
    seeds = boolean_input_array.shape
    X = boolean_input_array

    # The following bandwidth can be automatically detected using
    bandwidth = estimate_bandwidth(X, quantile=.3, n_samples=n_samples)

    ms = MeanShift(bandwidth=bandwidth * bandwidth_factor, cluster_all=True)
    ms.fit(X)

    labels = ms.labels_   # all labels
    cluster_centers = ms.cluster_centers_    # all clusters

    labels_unique = np.unique(labels)
    n_clusters_ = len(labels_unique)

    return ms

def mean_shift_predict(boolean_input_array, ms):
    """ Conduct mean shift analysis predictions on unlabeled data."""

    n_samples, n_features = boolean_input_array.shape
    X = boolean_input_array

    clusters_for_ea_sample = ms.predict(X)

    label_counts_dict = {}
    for lab in list(clusters_for_ea_sample):
        label_counts_dict[lab] = label_counts_dict.get(lab, 0)
        label_counts_dict[lab] += 1

    labels = ms.labels_
    cluster_centers = ms.cluster_centers_

    labels_unique = np.unique(labels)
    n_clusters_ = len(labels_unique)

    return clusters_for_ea_sample, cluster_centers

def train_model_and_pickle(n_train):
    """ Sample the data, extract features, train the model """

    profile_section = db.session.query(Profile.username, Profile.self_summary, Profile.message_me_if).all()

    random_to_train = sample(profile_section, n_train)

    # result is a much shorter list of tups. now extract only the usernames
    random_profiles_to_train = [item[0] for item in random_to_train]

    # which features to use, for cluster dimensions
    features_self_summary = get_words(Profile.self_summary, random_profiles_to_train,n_train)
    features_message_me_if = get_words(Profile.message_me_if, random_profiles_to_train, n_train)

    print "\nfeatures",features_self_summary
    print "\nfEADTRUES",features_message_me_if

    # must be the profiles that are trained
    text_list_self_summary_train = [item[1] for item in random_to_train]
    text_list_message_me_if_train = [item[2] for item in random_to_train]

    boolean_input_array_self_summary_train = convert(features_self_summary, text_list_self_summary_train)
    boolean_input_array_message_me_if_train = convert(features_message_me_if, text_list_message_me_if_train)

    # bandwidth is set here at 0.2
    ms_trained_object_self_summary = mean_shift_train(boolean_input_array_self_summary_train, 0.2)
    ms_trained_object_message_me_if = mean_shift_train(boolean_input_array_message_me_if_train, 0.2)

    joblib.dump(ms_trained_object_self_summary, 'model_pkl/self_summary_pickle.pkl')
    joblib.dump(ms_trained_object_message_me_if, 'model_pkl/message_me_if_pickle.pkl')

    file_features_self_summary = open('model_pkl/features_self_summary.txt', 'w')
    [file_features_self_summary.write(feature+"|") for feature in features_self_summary]
    file_features_self_summary.close()

    file_features_message_me_if = open('model_pkl/features_message_me_if.txt', 'w')
    [file_features_message_me_if.write(feature+"|") for feature in features_message_me_if]
    file_features_message_me_if.close()

    print "MODELS TRAINED AND SAVED!"
    return profile_section

def test_data():
    """ Finds and saves labels for users in databse """

    # get a list of all usernames
    all_usernames_tuple = db.session.query(Profile.username).all()
    all_usernames = [profile[0] for profile in all_usernames_tuple]
    total_num_users = len(all_usernames)
    increment = 1000
    start = 0
    count_users_both_labels = 0
    len_to_input = 0

    while start < total_num_users:
        end = start + increment

        get_users = all_usernames[start:end]
        profile_section = db.session.query(Profile.username, Profile.self_summary, Profile.message_me_if).filter(Profile.username.in_(get_users)).all()

        count, input_ = run_test_and_store(profile_section)
        count_users_both_labels += count
        len_to_input += input_
        start += increment

def run_test_and_store(profile_section):

    ms_trained_object_self_summary = joblib.load('model_pkl/self_summary_pickle.pkl')
    ms_trained_object_message_me_if = joblib.load('model_pkl/message_me_if_pickle.pkl')

    features_self_summary = open('model_pkl/features_self_summary.txt').read().rstrip("|").split("|")
    features_message_me_if = open('model_pkl/features_message_me_if.txt').read().rstrip("|").split("|")

    count_users_both_labels = 0

    sample_list = profile_section

    usernames = [item[0] for item in sample_list]

    # functions to attain labels for self_summary
    sample_list_self_summary = [item[1] for item in sample_list]
    boolean_input_array_self_summary = convert(features_self_summary, sample_list_self_summary)
    labels_self_summary, cluster_centers_self_summary = mean_shift_predict(boolean_input_array_self_summary, ms_trained_object_self_summary)

    # functions to attain labels for message_me_if
    sample_list_message_me_if = [item[2] for item in sample_list]
    boolean_input_array_message_me_if = convert(features_message_me_if, sample_list_message_me_if)
    labels_message_me_if, cluster_centers_message_me_if = mean_shift_predict(boolean_input_array_message_me_if, ms_trained_object_message_me_if)

    self_summary_dict = {}
    message_me_if_dict = {}

    for i in range(len(cluster_centers_self_summary)):
        self_summary_dict[labels_self_summary[i]] = zip(features_self_summary, cluster_centers_self_summary[i])

    for key, value in self_summary_dict.iteritems():
        for feature, center in value:
            if center == 1:
                new_self_summary = SelfSummaryLabel(self_summary_label=key, feature=feature)
                db.session.add(new_self_summary)

    for i in range(len(cluster_centers_message_me_if)):
        message_me_if_dict[labels_message_me_if[i]] = zip(features_message_me_if, cluster_centers_message_me_if[i])

    for key, value in message_me_if_dict.iteritems():
        for feature, center in value:
            if center == 1:
                new_message_me_if = MessageMeIfLabel(message_me_if_label=key, feature=feature)
                db.session.add(new_message_me_if)

    zipped =  zip(usernames, labels_self_summary, labels_message_me_if)
    to_input = list(zipped)
    for entry in to_input:
        username, self_summary_label, message_me_if_label = entry
        new_input = MeanShiftAlgo(username=username, self_summary_label=self_summary_label, message_me_if_label=message_me_if_label)

        if self_summary_label > 1 and message_me_if_label > 1:
            count_users_both_labels +=1

        db.session.add(new_input)

    db.session.commit()

    return count_users_both_labels, len(to_input)

if __name__ == "__main__":
    connect_to_db(app)

    # train_model_and_pickle(500)
    test_data()
    # print_cluster_words()
    # run_test_and_store()

max_stars_repo_name: adobs/project


text:
import cmath
import numpy
import math
import functools

from audio_to_midi import midi_writer, notes

class Converter(object):
    """
    An object which takes in a list of audio samples and transforms
        them into midi data which is written out to a .mid file.
    """

    def __init__(
        self,
        samples=None,
        samplerate=None,
        channels=0,
        time_window=None,
        activation_level=None,
        condense=None,
        note_count=None,
        outfile=None,
        progress_callback=None,
    ):
        """
        FFT object constructor.

        samples is a list of raw audio samples.
        rate is the sampling frequency of the audio samples.
        time_window is the interval (in ms) over which to compute the fft's.
        activation_level is the volume cutoff percentage for frequencies.
        outfile is the MIDI file to be written to.
        """

        self.samples = samples
        self.channels = channels
        self.length = len(samples)
        self.samplerate = samplerate
        self.time_window = time_window
        self.activation_level = activation_level
        self.condense = condense
        self.note_count = note_count
        self.outfile = outfile
        self.progress_callback = progress_callback
        self.notes = notes.generate()
        self.bpm = int((60 * 1000) / self.time_window)

        # Get the number of samples per time_window
        self.step_size = self.time_window_to_step_size(
            self.time_window, self.samplerate
        )

    def time_window_to_step_size(self, time_window, rate):
        """
        time_window is the time in ms over which to compute fft's.
        rate is the audio sampling rate in samples/sec.

        Transforms the time window into an index step size and
            returns the result.
        """

        # rate/1000(samples/ms) * time_window(ms) = step_size(samples)
        rate_per_ms = rate / 1000
        step_size = rate_per_ms * time_window

        return int(step_size)

    def reduce_freqs(self, freqs):
        """
        freqs is a list of amplitudes produced by fft_to_frequencies().

        Reduces the list of frequencies to a list of notes and their
            respective volumes by determining what note each frequency
            is closest to. It then reduces the list of amplitudes for each
            note to a single amplitude by summing them together.
        """

        reduced_freqs = {}
        for freq in freqs:
            for key, val in self.notes.items():
                key = key + 12
                # Find the freq's equivalence class, adding the amplitudes.
                if val[0] <= freq[0] <= val[2]:
                    if key in reduced_freqs.keys():
                        prev = reduced_freqs[key]
                        prev += freq[1]
                        reduced_freqs.update({key: prev})

                    else:
                        reduced_freqs.update({key: freq[1]})

        return reduced_freqs

    def freqs_to_midi(self, freq_list, channel):
        """
        freq_list is a list of frequencies with normalized amplitudes.

        Takes a list of notes and transforms the amplitude to a
            midi volume as well as adding track and channel info.
        """

        activation_level = self.activation_level / 100.0
        midi_list = []
        for freqs in freq_list:
            midi_notes = {}
            for key, val in freqs.items():
                if val >= activation_level:
                    # The key is the midi note.
                    midi_notes.update(
                        {
                            key: {
                                "track": 0,
                                "channel": channel,
                                "volume": int(127 * val),
                                "duration": 1,
                            }
                        }
                    )
            if self.note_count > 0:
                sorted_notes = sorted(
                    midi_notes, key=lambda val: midi_notes[val]["volume"]
                )[::-1]
                max_count = min(len(sorted_notes), self.note_count)
                midi_notes = {key: midi_notes[key] for key in sorted_notes[:max_count]}
            midi_list.append(midi_notes)

        return midi_list

    def fft_to_frequencies(self, amplitudes):
        """
        amplitudes is a list of amplitudes produced by the fft.

        Takes a list of amplitudes and transforms it into a list
            of midi notes by passing the list through reduce_freqs()
            and freqs_to_midi().
        """

        size = len(amplitudes)
        freqs = []
        # Determine the true amplitudes
        for i in range(size // 2):
            re = amplitudes[i].real
            im = amplitudes[i].imag

            amplitude = math.sqrt(re ** 2 + im ** 2)

            # Determine the frequency in Hz
            freq = i * float(self.samplerate) / size
            if freq > 20000.0:
                break
            else:
                freqs.append([freq, amplitude])

        # Transform the frequency info into midi compatible data.
        return self.reduce_freqs(freqs[1:])

    def normalize_freqs(self, freq_list):
        """
        freq list is a list of dicts containing all of the frequency
            data from the wav file.

        Normalizes the amplitudes of every frequency in every time step.
        """

        max_amplitude = 0.0
        for freqs in freq_list:
            for key, freq in freqs.items():
                if freq > max_amplitude:
                    max_amplitude = freq

        for freqs in freq_list:
            for key, amplitude in freqs.items():
                new_amplitude = amplitude / max_amplitude
                freqs.update({key: new_amplitude})

        return freq_list

    def condense_midi_notes(self, midi_list):
        """
        midi_list is a list of dicts containing midi compatible data.

        Combines consecutive notes accross time steps, using the maximum
            volume seen in the list as the resulting note's volume.
        """

        for i in range(len(midi_list)):
            if i < len(midi_list) - 1:
                cur_midi = midi_list[i]

                for note, info in cur_midi.items():
                    j = i + 1

                    while j < len(midi_list) - 1:
                        next_midi = midi_list[j]

                        if note in next_midi.keys():
                            if next_midi[note]["volume"] > cur_midi[note]["volume"]:
                                new_volume = next_midi[note]["volume"]

                            else:
                                new_volume = cur_midi[note]["volume"]
                            info.update(
                                {"duration": info["duration"] + 1, "volume": new_volume}
                            )
                            cur_midi.update({note: info})
                            next_midi.pop(note, None)

                        else:
                            break

                        j += 1

        return midi_list

    def convert(self):
        """
        Performs the fft for each time step and uses fft_to_frequencies
            to transform the result into midi compatible data. This data
            is then passed to a midi file writer to be written out.
        """

        steps = int(len(self.samples) / self.step_size)
        writer = midi_writer.MidiWriter(self.outfile, self.time_window, self.bpm)
        freqs = []

        samples = []
        for i in range(self.channels):
            samples.append([s[i] for s in self.samples])
        self.samples = samples

        current = 0
        total = self.channels * steps
        for channel in range(self.channels):
            freqs = []
            writer.reset_time()

            for i in range(steps):
                current += 1
                if self.progress_callback:
                    self.progress_callback(current, total)

                if i < steps - 1:
                    amplitudes = numpy.fft.fft(
                        self.samples[channel][
                            self.step_size * i : (self.step_size * i + self.step_size)
                        ]
                    )

                else:
                    amplitudes = numpy.fft.fft(
                        self.samples[channel][self.step_size * i :]
                    )

                freqs.append(self.fft_to_frequencies(amplitudes))

            freqs = self.normalize_freqs(freqs)
            if self.condense:
                midi_list = self.condense_midi_notes(self.freqs_to_midi(freqs, channel))
            else:
                midi_list = self.freqs_to_midi(freqs, channel)

            writer.add_notes(midi_list)

        writer.write_file()


text:
# Lint as: python3
# Copyright 2019 DeepMind Technologies Limited. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tests for ferminet.utils.scf."""

from typing import List, Tuple
from absl.testing import absltest
from absl.testing import parameterized
from ferminet.utils import scf
from ferminet.utils import system
import numpy as np
import pyscf

class ScfTest(parameterized.TestCase):

  def setUp(self):
    super(ScfTest, self).setUp()
    # disable use of temp directory in pyscf.
    # Test calculations are small enough to fit in RAM and we don't need
    # checkpoint files.
    pyscf.lib.param.TMPDIR = None

  @parameterized.parameters(
      {
          'molecule': [system.Atom('He', (0, 0, 0))],
          'nelectrons': (1, 1)
      },
      {
          'molecule': [system.Atom('N', (0, 0, 0))],
          'nelectrons': (5, 2)
      },
      {
          'molecule': [system.Atom('N', (0, 0, 0))],
          'nelectrons': (5, 3)
      },
      {
          'molecule': [system.Atom('N', (0, 0, 0))],
          'nelectrons': (4, 2)
      },
      {
          'molecule': [system.Atom('O', (0, 0, 0))],
          'nelectrons': (5, 3),
          'restricted': False,
      },
      {
          'molecule': [
              system.Atom('N', (0, 0, 0)),
              system.Atom('N', (0, 0, 1.4))
          ],
          'nelectrons': (7, 7)
      },
      {
          'molecule': [
              system.Atom('O', (0, 0, 0)),
              system.Atom('O', (0, 0, 1.4))
          ],
          'nelectrons': (9, 7),
          'restricted': False,
      },
  )
  def test_scf_interface(self,
                         molecule: List[system.Atom],
                         nelectrons: Tuple[int, int],
                         restricted: bool = True):
    """Tests SCF interface to a pyscf calculation.

    pyscf has its own tests so only check that we can run calculations over
    atoms and simple diatomics using the interface in ferminet.scf.

    Args:
      molecule: List of system.Atom objects giving atoms in the molecule.
      nelectrons: Tuple containing number of alpha and beta electrons.
      restricted: If true, run a restricted Hartree-Fock calculation, otherwise
        run an unrestricted Hartree-Fock calculation.
    """
    npts = 100
    xs = np.random.randn(npts, 3)
    hf = scf.Scf(molecule=molecule,
                 nelectrons=nelectrons,
                 restricted=restricted)
    hf.run()
    mo_vals = hf.eval_mos(xs)
    self.assertLen(mo_vals, 2)  # alpha-spin orbitals and beta-spin orbitals.
    for spin_mo_vals in mo_vals:
      # Evalute npts points on M orbitals/functions - (npts, M) array.
      self.assertEqual(spin_mo_vals.shape, (npts, hf._mol.nao_nr()))

if __name__ == '__main__':
  absltest.main()

max_stars_repo_name: llxlr/ferminet

max_issues_repo_name: llxlr/ferminet

max_forks_repo_name: llxlr/ferminet


text:
import numpy as np
import tensorflow as tf
import math

@tf.function
def groupLasso(y,p):

    inverted_y = -y + 1

    select_examples = tf.cast(tf.expand_dims(y, 1) * tf.expand_dims(inverted_y, 2), tf.float32)

    k = tf.subtract(tf.expand_dims(p, 2), tf.expand_dims(p, 1))
    k = tf.math.multiply(k, 2)
    k = tf.math.add(k, 1)
    k = tf.math.maximum(tf.constant(0, dtype=tf.float32), k)
    errors = tf.math.square(k)

    # Select instances
    errors = errors * select_examples

    # Calculate the loss_miss
    k = tf.reduce_sum(errors, 2)
    groups_miss = tf.math.sqrt(k)
    loss_miss = tf.reduce_sum(groups_miss, axis=1)

    # Choose the potential missing
    potential_missing = tf.argmax(groups_miss, axis=1)
    missing_high = tf.reduce_max(groups_miss, axis=1)

    # Calculate the loss_extra
    k = tf.reduce_sum(errors, 1)
    groups_extra = tf.math.sqrt(k)
    loss_extra = tf.reduce_sum(groups_extra, 1)

    # Choose potential extra
    potential_extra = tf.argmax(groups_extra, 1)
    extra_high = tf.reduce_max(groups_extra, 1)

    # Choose the missing or the extra
    CLASSES = tf.where(missing_high > extra_high, potential_missing, potential_extra)

    # Enter the loss for the sample
    loss = (loss_miss + loss_extra) / 2.0

    return loss, CLASSES

#@tf.function
def noForLasso(y,p):
    # Get the ones and zeros as ragged tensors

    bool_y = tf.cast(y, tf.bool)
    ones = tf.ragged.boolean_mask(p, bool_y)

    inverted_labels_bool = tf.math.logical_not(bool_y)
    zeros = tf.ragged.boolean_mask(p, inverted_labels_bool)

    # Stack corresponding ones and zeros on top of each other
    one_zero_pairs = tf.ragged.stack([ones,zeros], axis=1)

    # print(one_zero_pairs.shape)
    # print(one_zero_pairs)

    tf.RaggedTensor

    LOSSES = tf.TensorArray(dtype=tf.float32, size=0, dynamic_size=True)
    CLASSES = []

    #print(tf.subtract(tf.expand_dims(ones, axis=1), tf.expand_dims(zeros, axis=0)))
    #print(tf.subtract(tf.expand_dims(ones[0], axis=1), tf.expand_dims(zeros[0], axis=0)))

    # print(tf.subtract(tf.expand_dims(one_zero_pairs[:, 0], axis=0), tf.expand_dims(one_zero_pairs[:,1], axis=1)))
    # print(tf.subtract(tf.expand_dims(zeros, axis=2), tf.expand_dims(ones, axis=0)).shape)
    # print(tf.subtract(tf.expand_dims(zeros, axis=1), tf.expand_dims(ones, axis=0)).shape)

    '''
    # Calculate missing class errors
    onez = tf.expand_dims(one_zero_pair[0], axis=0)
    zeroz = tf.expand_dims(one_zero_pair[1], axis=1)
    k = tf.math.subtract(zeroz, onez)
    k = tf.math.multiply(k, 2)
    k = tf.math.add(k, 1)
    k = tf.math.maximum(tf.constant(0, dtype=tf.float32), k)
    errors_miss = tf.math.square(k)

    # Calculate extra class errors
    onez = tf.expand_dims(one_zero_pair[0], axis=1)
    zeroz = tf.expand_dims(one_zero_pair[1], axis=0)
    k = tf.math.subtract(zeroz, onez)
    k = tf.math.multiply(k, 2)
    k = tf.math.add(k, 1)
    k = tf.math.maximum(tf.constant(0, dtype=tf.float32), k)
    errors_extra = tf.math.square(k)

    # Calculate the loss_miss
    k = tf.reduce_sum(errors_miss, axis=1)
    groups_miss = tf.math.sqrt(k)
    loss_miss = tf.reduce_sum(groups_miss)

    # Calculate the extra_miss
    k = tf.reduce_sum(errors_extra, axis=1)
    groups_extra = tf.math.sqrt(k)
    loss_extra = tf.reduce_sum(groups_extra)

    # Enter the loss for the sample
    loss = (loss_miss + loss_extra) / 2.0
    '''

    return LOSSES, CLASSES

def calc_errors(one_zero_pairs):
    # Calculate missing class errors
    onez = tf.expand_dims(one_zero_pairs[0], axis=0)
    zeroz = tf.expand_dims(one_zero_pairs[1], axis=1)
    k = tf.math.subtract(zeroz, onez)
    k = tf.math.multiply(k, 2)
    k = tf.math.add(k, 1)
    k = tf.math.maximum(tf.constant(0, dtype=tf.float32), k)
    errors_miss = tf.math.square(k)

    # Calculate extra class errors
    onez = tf.expand_dims(one_zero_pairs[0], axis=1)
    zeroz = tf.expand_dims(one_zero_pairs[1], axis=0)
    k = tf.math.subtract(zeroz, onez)
    k = tf.math.multiply(k, 2)
    k = tf.math.add(k, 1)
    k = tf.math.maximum(tf.constant(0, dtype=tf.float32), k)
    errors_extra = tf.math.square(k)

    # Calculate the loss_miss
    k = tf.reduce_sum(errors_miss, axis=1)
    groups_miss = tf.math.sqrt(k)
    loss_miss = tf.reduce_sum(groups_miss)

    # Calculate the extra_miss
    k = tf.reduce_sum(errors_extra, axis=1)
    groups_extra = tf.math.sqrt(k)
    loss_extra = tf.reduce_sum(groups_extra)

    # Enter the loss for the sample
    return (loss_miss + loss_extra) / 2.0

def onlyFasterLasso(y,p):
    # Get the ones and zeros as ragged tensors
    bool_y = tf.cast(y, tf.bool)
    ones = tf.ragged.boolean_mask(p, bool_y)

    inverted_labels_bool = tf.math.logical_not(bool_y)
    zeros = tf.ragged.boolean_mask(p, inverted_labels_bool)

    # Stack corresponding ones and zeros on top of each other
    one_zero_pairs = tf.ragged.stack([ones,zeros], axis=1)

    CLASSES = []
    LOSSES = tf.map_fn(calc_errors, one_zero_pairs, fn_output_signature=tf.float32, parallel_iterations=32)

    return LOSSES, CLASSES

@tf.function
def onlyLasso(y,p):
    # Get the ones and zeros as ragged tensors
    bool_y = tf.cast(y, tf.bool)
    ones = tf.ragged.boolean_mask(p, bool_y)

    inverted_labels_bool = tf.math.logical_not(bool_y)
    zeros = tf.ragged.boolean_mask(p, inverted_labels_bool)

    # Stack corresponding ones and zeros on top of each other
    one_zero_pairs = tf.ragged.stack([ones,zeros], axis=1)

    indVar = 0
    LOSSES = tf.TensorArray(dtype=tf.float32, size=0, dynamic_size=True)
    CLASSES = []

    # Fix this if you can!
    for one_zero_pair in one_zero_pairs:
        # Calculate missing class errors
        onez = tf.expand_dims(one_zero_pair[0], axis=0)
        zeroz = tf.expand_dims(one_zero_pair[1], axis=1)
        k = tf.math.subtract(zeroz, onez)
        k = tf.math.multiply(k, 2)
        k = tf.math.add(k, 1)
        k = tf.math.maximum(tf.constant(0, dtype=tf.float32), k)
        errors_miss = tf.math.square(k)

        # Calculate extra class errors
        onez = tf.expand_dims(one_zero_pair[0], axis=1)
        zeroz = tf.expand_dims(one_zero_pair[1], axis=0)
        k = tf.math.subtract(zeroz, onez)
        k = tf.math.multiply(k, 2)
        k = tf.math.add(k, 1)
        k = tf.math.maximum(tf.constant(0, dtype=tf.float32), k)
        errors_extra = tf.math.square(k)

        # Calculate the loss_miss
        k = tf.reduce_sum(errors_miss, axis=1)
        groups_miss = tf.math.sqrt(k)
        loss_miss = tf.reduce_sum(groups_miss)

        # Calculate the extra_miss
        k = tf.reduce_sum(errors_extra, axis=1)
        groups_extra = tf.math.sqrt(k)
        loss_extra = tf.reduce_sum(groups_extra)

        # Enter the loss for the sample
        LOSSES = LOSSES.write(indVar, (loss_miss + loss_extra) / 2.0)

        indVar += 1

    return LOSSES.stack(), CLASSES

@tf.function
def fasterLasso(y,p):
    '''
    input: predictions from DNN as y. Its shape is (batch_size, num_of_classes)
           labels of the predicted samples. Its shape is (batch_size, num_of_classes)
    output: LOSSES tensor that has (batch_size) errors for each sample
            CLASSES tensor that has the index of the potential noisy class for each sample.

    Group Lasso Error for missing classes: E(zero_classes) sqrt( E(one_classes) F(K,L)^2 )
    Group Lasso Error for extra classes: E(one_classes) sqrt( E(zero_classes) F(K,L)^2 )
    E(x) is the cumulative sum!
    THE DISCRIMINATIVE FUNCTION IS F(K,L) = MAX(0, 2*(L-K)+1)
    ARTIFICIAL TRESHHOLD IS 0.5 FOR BOTH OF THEM
    For the true classification, the function gives -1
    For a missing class or an extra class, the function gives +1
    And for the extreme, which is missing class and extra class together, the function gives +3

    How to figure if Missing class or Extra class?
    Look at the respective error group if all the elements of the subarray are non-zero, this
    element is missing class or extra class respectively. However, this method does not give
    always the right answers. So if there are more than one missing classes or extra labels just
    choose one with the biggest loss.

    YOU CANT MEASURE HOW GOOD YOU ARE DOING COMPARED TO THE TRUE LABEL OF A SAMPLE!
    '''
    # Get the ones and zeros as ragged tensors. For each sample the number of ones and zeros are different. That's why it is a ragged tensor
    bool_y = tf.cast(y, tf.bool) # Transform the labels into a boolean tensor to get positive classes
    ones = tf.ragged.boolean_mask(p, bool_y)

    inverted_labels_bool = tf.math.logical_not(bool_y)
    bool_y = tf.cast(y, tf.bool) # Transform the labels into a boolean tensor to get negative  classes
    zeros = tf.ragged.boolean_mask(p, inverted_labels_bool)

    # This includes positive and negative class pairs for every sample in the batch
    one_zero_pairs = tf.ragged.stack([ones,zeros], axis=1) # Stack corresponding ones and zeros on top of each other

    indVar = 0 # Increment this value to write loss and class values into the corresponding tensor arrays
    LOSSES = tf.TensorArray(dtype=tf.float32, size=0, dynamic_size=True)
    CLASSES = tf.TensorArray(dtype=tf.int32, size=0, dynamic_size=True)

    # Fix this if you can!
    for one_zero_pair in one_zero_pairs:
        # Calculate missing class errors
        onez = tf.expand_dims(one_zero_pair[0], axis=0) # Expand dims to do pairwise calculations
        zeroz = tf.expand_dims(one_zero_pair[1], axis=1)
        k = tf.math.subtract(zeroz, onez)
        k = tf.math.multiply(k, 2)
        k = tf.math.add(k, 1)
        k = tf.math.maximum(tf.constant(0, dtype=tf.float32), k)
        errors_miss = tf.math.square(k)
        # Calculate extra class errors
        onez = tf.expand_dims(one_zero_pair[0], axis=1) # Expand dims to do pairwise calculations
        zeroz = tf.expand_dims(one_zero_pair[1], axis=0)
        k = tf.math.subtract(zeroz, onez)
        k = tf.math.multiply(k, 2)
        k = tf.math.add(k, 1)
        k = tf.math.maximum(tf.constant(0, dtype=tf.float32), k)
        errors_extra = tf.math.square(k)

        # Calculate the loss_miss
        k = tf.reduce_sum(errors_miss, axis=1)
        groups_miss = tf.math.sqrt(k)
        loss_miss = tf.reduce_sum(groups_miss)
        # Calculate the extra_miss
        k = tf.reduce_sum(errors_extra, axis=1)
        groups_extra = tf.math.sqrt(k)
        loss_extra = tf.reduce_sum(groups_extra)

        # Find the noisy missing class, choose the one with the biggest loss
        # Index that I get is not the real index of the missing class. It is the index of it in its own group, which is zeros!
        # Find the noisy extra class, choose the one with the biggest loss
        # Index that I get is not the real index of the missing class. It is the index of it in its own group, which is zeros!
        onez_inds = tf.where(bool_y[0])
        zeroz_inds = tf.where(tf.math.logical_not(bool_y[0]))

        row_wise_min_miss = tf.reduce_min(errors_miss, axis=1) # Get the min element in the errors for classes to figure if all the errors are non-zero
        row_wise_min_extra = tf.reduce_min(errors_extra, axis=1)
        indices_miss = tf.where(tf.math.not_equal(row_wise_min_miss, 0))
        indices_extra = tf.where(tf.math.not_equal(row_wise_min_extra, 0))
        potential_missing_classes = tf.gather(groups_miss, indices_miss)
        potential_extra_classes = tf.gather(groups_extra, indices_extra)

        try:
            potential_missing = tf.math.argmax(potential_missing_classes)
            missing_class_index = tf.gather(indices_miss, potential_missing)
        except:
            missing_class_index = None
        try:
            potential_extra = tf.math.argmax(potential_extra_classes)
            extra_class_index = tf.gather(indices_extra, potential_extra)
        except:
            extra_class_index = None

        # Enter the loss for the sample
        LOSSES = LOSSES.write(indVar, (loss_miss + loss_extra) / 2.0)
        CLASSES = CLASSES.write(indVar, missing_class_index)

        indVar += 1

    return LOSSES.stack(), CLASSES.stack()

def bucakLasso(y,p):

    groups_miss = []
    errors_miss = []
    loss_miss = 0
    missing_classes = set()

    for i, l in np.ndenumerate(p):
        if y[i] == -1:
            group = 0
            errors = []
            for idx, k in np.ndenumerate(p):
                if y[idx] == 1:
                    error = (max(0, (l-k)+1) ** 2)
                    group += error
                    errors.append(error)
            groups_miss.append(math.sqrt(group))
            missing_classes.add(i[0])
            errors_miss.append(errors)

    for i in groups_miss:
        loss_miss += i

    return loss_miss, groups_miss, errors_miss, missing_classes

def calc_missing(p, y):
    groups_miss = []
    errors_miss = []
    loss_miss = 0
    missing_classes = []

    for i, l in enumerate(p):
        if y[i] == 0:
            group = 0
            errors = []
            for idx, k in enumerate(p):
                if y[idx] == 1:
                    error = (max(0, 2*(l-k)+1) ** 2)
                    group += error
                    errors.append(error)

            # This is just to find the missing labels
            if all(e != 0 for e in errors):
                missing_classes.append([math.sqrt(group), i])

            groups_miss.append(math.sqrt(group))
            errors_miss.append(errors)

    for i in groups_miss:
        loss_miss += i

    # Choose the one with biggest loss
    try:
        missing_class = max(missing_classes)
    except:
        missing_class = None

    return groups_miss, errors_miss, loss_miss, missing_classes, missing_class

def calc_extra(p, y):
    groups_extra = []
    errors_extra = []
    loss_extra = 0
    extra_classes = []

    for i, k in enumerate(p):
        if y[i] == 1:
            group = 0
            errors = []
            for idx, l in enumerate(p):
                if y[idx] == 0:
                    error = (max(0, 2*(l-k)+1) ** 2)
                    errors.append(error)
                    group += error

            # This is just to find the extra labels
            if all(e != 0 for e in errors):
                extra_classes.append([math.sqrt(group), i])

            groups_extra.append(math.sqrt(group))
            errors_extra.append(errors)

    for i in groups_extra:
        loss_extra += i

    # Choose the one with biggest loss
    try:
        extra_class = max(extra_classes)
    except:
        extra_class = None

    return groups_extra, errors_extra, loss_extra, extra_classes, extra_class

def oldLasso(ys,ps):

    '''
    THE DISCRIMINATIVE FUNCTION IS F(K,L) = MAX(0, 2*(L-K)+1)
    ARTIFICIAL TRESHHOLD IS 0.5 FOR BOTH OF THEM
    For the true classification, the function gives -1
    For a missing class or an extra class, the function gives +1
    And for the extreme, which is missing class and extra class together, the function gives +3

    How to figure if Missing class or Extra class?
    Look at the respective error group if all the elements of the subarray are non-zero, this
    element is missing class or extra class respectively. However, this method does not give
    always the right answers. So if there are more than one missing classes or extra labels just
    choose one with the biggest loss.

    YOU CANT MEASURE HOW GOOD YOU ARE DOING COMPARED TO THE TRUE LABEL OF A SAMPLE!
    '''

    LOSSES = []
    CLASSES = []
    results = []

    for y, p in zip(ys,ps):

        # FIND THE MISSING CLASS LABELS
        groups_miss, errors_miss, loss_miss, missing_classes, missing_class = calc_missing(p, y)
        print(f"errors_miss: {errors_miss}")

        # TO FIND THE EXTRA CLASS LABELS
        groups_extra, errors_extra, loss_extra, extra_classes, extra_class = calc_extra(p,y)

        LOSS = (loss_miss + loss_extra) / 2.0

        if missing_class == None and extra_class == None:
            change_class = None
        elif missing_class == None and extra_class != None:
            change_class = extra_class[1]
        elif missing_class != None and extra_class == None:
            change_class = missing_class[1]
        elif missing_class[0] > extra_class[0]:
            change_class = missing_class[1]
        else:
            change_class = extra_class[1]

        LOSSES.append(LOSS)
        CLASSES.append(change_class)

        results.append([LOSS, change_class, missing_class, extra_class, loss_miss, groups_miss, errors_miss, loss_extra, groups_extra, errors_extra, missing_classes, extra_classes])

    return np.array(LOSSES), np.array(CLASSES)

def test():
    y1 = np.array([[1,0,1,0,1,1,0,0,1]]) # 9 CLASSES
    #noisy_y1 = np.array([[1,0,0,0,1,0,0,0,1]]) # Missing classes at indexes 2 and 5
    #noisy_y1 = np.array([[1,0,1,1,1,1,0,1,1]]) # Extra classes at indexes 3 and 7
    noisy_y1 = np.array([[1,0,0,0,1,1,0,1,1]]) # Missing class at index 2 and extra class at index 7
    double_noisy_y1 = np.array([[1,0,0,0,1,0,0,1,1],[1,0,0,0,1,1,0,1,1]]) # Missing class at index 2 and extra class at index 7
    p1 = np.array([[0.91,0.11,0.87,0.13,0.92,0.83,0.12,0.05,0.95]])
    p2 = np.array([[0.78,0.34,0.72,0.2,0.86,0.62,0.4,0.17,0.95]])
    p3 = np.array([[0.53,0.47,0.58,0.39,0.51,0.54,0.49,0.46,0.61]])
    p4 = np.array([[0.91,0.11,0.87,0.13,0.92,0.83,0.12,0.05,0.95],[0.78,0.34,0.72,0.2,0.86,0.62,0.4,0.17,0.95]], dtype=np.float32)
    losses, classes = newLasso(double_noisy_y1,p4)
    losses1, classes1 = groupLasso(double_noisy_y1,p4)

    print(f"losses: {losses}")
    print(f"classes: {classes}")
    print(f"losses1: {losses1}")

    '''
    results = groupLasso(double_noisy_y1,p4)
    print(f"LOSS1: {results[0][0]}")
    print(f"change_class: {results[0][1]}")
    print(f"missing_class: {results[0][2]}")
    print(f"extra_class: {results[0][3]}")
    print(f"loss_miss: {results[0][4]}")
    print(f"groups_miss: {results[0][5]}")
    print(f"errors_miss: {results[0][6]}")
    print(f"loss_extra: {results[0][7]}")
    print(f"groups_extra: {results[0][8]}")
    print(f"errors_extra: {results[0][9]}")
    print(f"missing_classes: {results[0][10]}")
    print(f"extra_classes: {results[0][11]}")
    '''

def bucakTest():
    y1 = np.array([1,-1,1,-1,1,1,-1,-1,1]) # 9 CLASSES
    noisy_y1 = np.array([1,-1,-1,-1,1,-1,-1,-1,1]) # Missing classes at indexes 2 and 5
    #noisy_y1 = np.array([1,-1,1,1,1,1,-1,1,1]) # Extra classes at indexes 3 and 7
    #noisy_y1 = np.array([1,-1,-1,-1,1,1,-1,1,1]) # Missing class at index 2 and extra class at index 7
    p1 = np.array([0.91,-0.88,0.87,-0.83,0.92,0.83,-0.93,-0.91,0.95])  # loss = 3.1990626168794223
    p2 = np.array([0.78,-0.68,0.72,-0.8,0.86,0.62,-0.32,-0.56,0.95])   # loss = 2.804749814873396
    p3 = np.array([0.23,-0.32,0.42,-0.52,0.27,0.37,-0.32,-0.13,0.19])   # loss = 7.14685765636982
    p4 = np.array([0.,0.,0.,0.,0.,0.,0.,0.,0.])   # loss = 10.392304845413262
    p5 = np.array([1.,-1.,1.,-1.,1.,1.,-1.,-1.,1.])   # loss = 3.4641016151377544
    p6 = np.array([1.,-1.,-1.,-1.,1.,-1.,-1.,-1.,1.])   # loss = 0.0
    p7 = np.array([.02,-.02,.02,-.02,.02,.02,-.02,-.02,.02])
    p8 = np.array([.4,-.4,.4,-.4,.4,.4,-.4,-.4,.4])
    loss_miss1, groups_miss1, errors_miss1, missing_classes1 = bucakLasso(y1,p8)
    print(f"loss_miss1: {loss_miss1}")
    print(f"groups_miss1: {groups_miss1}")
    print(f"errors_miss1: {errors_miss1}")
    print(f"missing_classes1: {missing_classes1}")

# test()
bucakTest()

max_stars_repo_name: turguthaspolat/CCML

max_issues_repo_name: turguthaspolat/CCML

max_forks_repo_name: turguthaspolat/CCML


text:
"""Util module."""
import numpy as np

def is_resign_act(board_size, action):
    """Check if the action is resign."""
    return action == board_size ** 2

def is_pass_act(board_size, action):
    """Check if the action is pass."""
    return action == board_size ** 2 + 1

def get_possible_acts(board, plr_color):
    """Get possible actions."""
    actions = []
    _d = board.shape[-1]
    opp_color = 1 - plr_color
    for pos_x in range(_d):
        for pos_y in range(_d):
            if (board[2, pos_x, pos_y] == 0):
                continue
            for dx in [-1, 0, 1]:
                for dy in [-1, 0, 1]:
                    if(dx == 0 and dy == 0):
                        continue
                    nx = pos_x + dx
                    ny = pos_y + dy
                    n = 0
                    if (nx not in range(_d) or ny not in range(_d)):
                        continue
                    while(board[opp_color, nx, ny] == 1):
                        tmp_nx = nx + dx
                        tmp_ny = ny + dy
                        if (tmp_nx not in range(_d)
                                or tmp_ny not in range(_d)):
                            break
                        n += 1
                        nx += dx
                        ny += dy
                    if(n > 0 and board[plr_color, nx, ny] == 1):
                        actions.append(pos_x * _d + pos_y)
    if len(actions) == 0:
        actions = [_d**2 + 1]
    return actions

def valid_reverse_opp(board, coords, plr_color):
    """Check if there is a reversible location."""
    _d = board.shape[-1]
    opp_color = 1 - plr_color
    pos_x, pos_y = coords[:2]
    for dx in [-1, 0, 1]:
        for dy in [-1, 0, 1]:
            if(dx == 0 and dy == 0):
                continue
            nx = pos_x + dx
            ny = pos_y + dy
            _n = 0
            if (nx not in range(_d) or ny not in range(_d)):
                continue
            while(board[opp_color, nx, ny] == 1):
                tmp_nx = nx + dx
                tmp_ny = ny + dy
                if (tmp_nx not in range(_d) or tmp_ny not in range(_d)):
                    break
                _n += 1
                nx += dx
                ny += dy
            if(_n > 0 and board[plr_color, nx, ny] == 1):
                return True
    return False

def is_valid(board, action, plr_color):
    """Check valid place."""
    coords = act2coord(board, action)
    if board[2, coords[0], coords[1]] == 1:
        if valid_reverse_opp(board, coords, plr_color):
            return True
        else:
            return False
    else:
        return False

def make_place(board, action, plr_color):
    """Make palce."""
    coords = act2coord(board, action)

    _d = board.shape[-1]
    opp_color = 1 - plr_color
    pos_x, pos_y = coords[:2]

    for dx in [-1, 0, 1]:
        for dy in [-1, 0, 1]:
            if(dx == 0 and dy == 0):
                continue
            nx = pos_x + dx
            ny = pos_y + dy
            _n = 0
            if (nx not in range(_d) or ny not in range(_d)):
                continue
            while(board[opp_color, nx, ny] == 1):
                tmp_nx = nx + dx
                tmp_ny = ny + dy
                if (tmp_nx not in range(_d) or tmp_ny not in range(_d)):
                    break
                _n += 1
                nx += dx
                ny += dy
            if(_n > 0 and board[plr_color, nx, ny] == 1):
                nx = pos_x + dx
                ny = pos_y + dy
                while(board[opp_color, nx, ny] == 1):
                    board[2, nx, ny] = 0
                    board[plr_color, nx, ny] = 1
                    board[opp_color, nx, ny] = 0
                    nx += dx
                    ny += dy
                board[2, pos_x, pos_y] = 0
                board[plr_color, pos_x, pos_y] = 1
                board[opp_color, pos_x, pos_y] = 0
    return board

def coord2act(board, coords):
    """Convert coordinate to action."""
    return coords[0] * board.shape[-1] + coords[1]

def act2coord(board, action):
    """Convert action to coordinate."""
    return action // board.shape[-1], action % board.shape[-1]

def is_finished(board):
    """Chack game is finished."""
    _d = board.shape[-1]

    plr_score_x, _ = np.where(board[0, :, :] == 1)
    plr_score = len(plr_score_x)
    opp_score_x, _ = np.where(board[1, :, :] == 1)
    opp_score = len(opp_score_x)
    if plr_score == 0:
        return -1
    elif opp_score == 0:
        return 1
    else:
        free_x, free_y = np.where(board[2, :, :] == 1)
        if free_x.size == 0:
            if plr_score > (_d**2)/2:
                return 1
            elif plr_score == (_d**2)/2:
                return 1
            else:
                return -1
        else:
            if (get_possible_acts(board, 0) == [board.shape[-1] ** 2 + 1])\
             and (get_possible_acts(board, 0) == [board.shape[-1] ** 2 + 1]):
                if plr_score > opp_score:
                    return 1
                else:
                    return -1
            return 0
    return 0


text:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import json
import os

alias = {}
styles = {}

def parse_index_name(iname):
    # parse '*.map-DFS-3-c4-inv' to itype="inv", d=4, hlevel=3
    # or '*.map-DFS-3-c4' to itype="fwd", d=4, hlevel=3
    ps = iname.split('-')
    itype = "forward"
    if ps[-1] == "inv":
        itype = "backward"
        ps.pop()
    r = int(ps[-1][1:]) if ps[-1][0] == 'c' else 0
    ps.pop()
    ps.pop()
    ps.pop()
    mname = '-'.join(ps)[:-4]
    return dict(map=mname,itype=itype,r=r)

def load_size(fname, filter=True):
    with open(fname, 'r') as f:
        raw = f.readlines()
    rows = []
    game_maps = os.listdir("../maps/gppc")
    header = ["map", "itype", "r", "size"]
    for line in raw:
        size, path = line.split()
        data = parse_index_name(path)
        mapname = data['map'].split('/')[-1]
        if not filter or (mapname + ".map") in game_maps:
            rows.append([mapname, data['itype'], data['r'], size])
    df = pd.DataFrame.from_records(rows, columns=header)
    df = df[header].apply(pd.to_numeric, errors="ignore")
    return df

def load_data(fname):
    with open(fname, "r") as f:
        # ignore `vertices: ...` at head and `Total: ..` at tail
        raws = f.readlines()
    header = [i.strip() for i in raws[0].split(',')]
    lines = [[j.strip() for j in i.split(',')] for i in raws[1:]]
    t = pd.DataFrame.from_records(lines, columns=header)
    res = t[header].apply(pd.to_numeric, errors='ignore')
    return res

def load_files(paths):
    frames = []
    for i in paths:
        print(i)
        frames.append(load_data(i))
    res = pd.concat(frames)
    return res

def gen_xy(df=None, colx='', coly='', ignore=True, limit=20, aggregate=np.average):
    tg = df.groupby(colx)
    x = []
    y = []
    for k, v in tg[coly].apply(lambda _: aggregate(_)).items():
        if ignore and tg.size()[int(k)] < limit:
            continue
        x.append(int(k))
        y.append(v)
    return x, y

def plot_graph(xlabel='', ylabel='', xs=[[]], ys=[[]], labels=[], color=None,
               yscale='log', xscale=None, ylim=None, xlim=None, saveto=None, xticks=None, yticks=None,
               loc='out', **kw):

    fig, ax = plt.subplots()
    ax.set_xlabel(xlabel)
    ax.set_ylabel(ylabel)
    ax.set_yscale(yscale)
    font = {'family': 'monospace',
            'weight': 'bold',
            'size': 22}
    plt.rc('font',**font)
    plt.rc("text", usetex=True)
    if xscale is not None:
      ax.set_xscale(xscale)
    if ylim is not None:
        ax.set_ylim(ylim)
    if xlim is not None:
        ax.set_xlim(xlim)
    if xticks is not None:
      print(xticks)
      plt.xticks(xticks)
    if yticks is not None:
      plt.yticks(yticks)

    n = len(xs)
    for i in range(n):
        x = xs[i]
        y = ys[i]
        if styles.get(labels[i]) is not None:
            plt.plot(x, y, styles[labels[i]], label=labels[i], markersize=5)
        else:
            # ax.scatter(x, y)
            ax.plot(x, y, label=labels[i])
    ax.legend(labels)
    if loc == 'in':
      ax.legend(loc='best', fancybox=True, framealpha=0, prop={'size': 14})
    else:
      bbox_to_anchor = kw.get("bbox_to_anchor", (0.5, 1.4))
      ax.legend(loc='upper center', bbox_to_anchor=bbox_to_anchor, ncol=4, fancybox=True, prop={'size': 16})
    plt.grid(True)
    if saveto is not None:
        fig.savefig(saveto, bbox_inches='tight')

max_stars_repo_name: eggeek/CPD-Hsymbol-Wildcard

max_issues_repo_name: eggeek/CPD-Hsymbol-Wildcard

max_forks_repo_name: eggeek/CPD-Hsymbol-Wildcard


text:
import argparse
import json
import os
import kindred
import random
import pickle
from collections import defaultdict
from sklearn.linear_model import LogisticRegression
import numpy as np
import math
import time
import random
import itertools
import pgmine

def entitiesOverlap(candidateRelation):
	for e1,e2 in itertools.combinations(candidateRelation.entities,2):
		for start1,end1 in e1.position:
			for start2,end2 in e2.position:
				if not (end1 <= start2 or end2 <= start1 ):
					return True

	return False

if __name__ == '__main__':
	parser = argparse.ArgumentParser(description='Merge sentence data and do some annotation')
	parser.add_argument('--mode',required=True,type=str,help='Whether to focus on star_allele, rs or other')
	parser.add_argument('--selectedChemicals',required=True,type=str,help='Which chemicals to filter for')
	parser.add_argument('--sentenceData',required=True,type=str,help='Directory with sentence data')
	parser.add_argument('--variantStopwords',required=True,type=str,help='Mutations to remove')
	parser.add_argument('--outDir',required=True,type=str,help='Directory to dump standoff format corpus')
	parser.add_argument('--fileCount',required=False,type=int,help='Optionally limit the number of BioC file to load')
	parser.add_argument('--filterTerms',type=str,help='Optional file of filter terms')
	args = parser.parse_args()

	start = time.time()

	assert args.mode in ['star_allele','rs','star_rs','other']

	with open(args.selectedChemicals) as f:
		drugData = json.load(f)
		drugMeshIDs = set([ 'MESH:'+d['MeSH'] for d in drugData ])

	filterTerms = set()
	if args.filterTerms:
		with open(args.filterTerms) as f:
			for line in f:
				filterTerms.add(line.strip().lower())

	with open(args.variantStopwords) as f:
		variantStopwords = set( line.strip().lower() for line in f )

	stopwords = {'dopamine','insulin','caffeine','nicotine','choline'}
	print("Loading sentences...")
	corpus = kindred.Corpus()
	filenames = sorted(os.listdir(args.sentenceData))

	if args.fileCount:
		filenames = random.sample(filenames, args.fileCount)

	for i,filename in enumerate(filenames):
		print(i, filename)

		tmpCorpus = kindred.load('biocxml',os.path.join(args.sentenceData,filename))
		if args.filterTerms:
			tmpCorpus.documents = [ doc for doc in tmpCorpus.documents if any ( filterTerm in doc.text.lower() for filterTerm in filterTerms ) ]
		tmpCorpus.documents = [ doc for doc in tmpCorpus.documents if not any ( stopword in doc.text.lower() for stopword in stopwords ) ]

		filtered = []
		for doc in tmpCorpus.documents:
			if args.mode == 'star_allele':
				doc.entities = [ e for e in doc.entities if not (e.entityType == 'Mutation' and not e.text.strip().startswith('*')) ]
			elif args.mode == 'rs':
				doc.entities = [ e for e in doc.entities if not (e.entityType == 'Mutation' and not e.text.strip().startswith('rs')) ]
			elif args.mode == 'star_rs':
				doc.entities = [ e for e in doc.entities if not (e.entityType == 'Mutation' and not (e.text.strip().startswith('rs') or e.text.strip().startswith('*'))) ]
			else:
				doc.entities = [ e for e in doc.entities if not (e.entityType == 'Mutation' and (e.text.strip().startswith('*') or e.text.strip().startswith('rs'))) ]
				#doc.entities = [ e for e in doc.entities if not (e.entityType == 'Mutation' and not e.metadata['conceptid'].startswith('rs')) ]

			doc.entities = [ e for e in doc.entities if e.position[0][0] >= 0 ]
			doc.entities = [ e for e in doc.entities if e.position[0][1] <= len(doc.text) ]
			doc.entities = [ e for e in doc.entities if not (e.entityType == 'Chemical' and not e.metadata['conceptid'] in drugMeshIDs) ]
			doc.entities = [ e for e in doc.entities if not (e.entityType == 'Chemical' and len(e.text) <= 4) ]
			doc.entities = [ e for e in doc.entities if not (e.entityType == 'Mutation' and pgmine.normalizeMutation(e.text) is None) ]
			doc.entities = [ e for e in doc.entities if not (e.entityType == 'Mutation' and e.text.lower() in variantStopwords) ]

			entityTypes = set( e.entityType for e in doc.entities )
			if 'Chemical' in entityTypes and 'Mutation' in entityTypes:
				filtered.append(doc)

		corpus.documents += filtered

	print("Found: ", len(corpus.documents))
	corpus.documents = random.sample(corpus.documents, 500)

	kindred.save(corpus,'standoff',args.outDir)
	kindred.save(corpus,'biocxml',os.path.join(args.outDir,'corpus.bioc.xml'))

max_stars_repo_name: usametov/pgxmine

max_issues_repo_name: usametov/pgxmine

max_forks_repo_name: usametov/pgxmine