text:
import argparse
import itertools
import json
import os

import chainer
import numpy as np
import nibabel as nib
import pandas as pd

from model import VoxResNet
from utils import crop_patch, load_nifti, feedforward

def main():
    parser = argparse.ArgumentParser(
        description="calculate class probabilities with VoxResNet")
    parser.add_argument(
        "--input_file", "-i", type=str,
        help="input json file of test dataset"
    )
    parser.add_argument(
        "--output_suffix", "-o", type=str,
        help="result of the segmentation"
    )
    parser.add_argument(
        "--model", "-m", type=str,
        help="a file containing parameters of trained VoxResNet"
    )
    parser.add_argument(
        "--input_shape", type=int, nargs="*", action="store",
        default=[80, 80, 80],
        help="input patch shape of VoxResNet, default=[80, 80, 80]"
    )
    parser.add_argument(
        "--output_shape", type=int, nargs="*", action="store",
        default=[60, 60, 60],
        help="output patch shape of VoxResNet, default=[60, 60, 60]"
    )
    parser.add_argument(
        "--gpu", "-g", default=-1, type=int,
        help="negative value indicates no gpu, default=-1"
    )
    parser.add_argument(
        "--n_tiles", type=int, nargs="*", action="store",
        default=[5, 5, 5],
        help="number of tiles along each axis"
    )
    args = parser.parse_args()
    print(args)

    with open(args.input_file) as f:
        dataset = json.load(f)
    test_df = pd.DataFrame(dataset["data"])

    vrn = VoxResNet(dataset["in_channels"], dataset["n_classes"])
    chainer.serializers.load_npz(args.model, vrn)

    if args.gpu >= 0:
        chainer.cuda.get_device_from_id(args.gpu).use()
        vrn.to_gpu()

    for image_path, subject in zip(test_df["image"], test_df["subject"]):
        image, affine = load_nifti(image_path, with_affine=True)
        output = feedforward(
            vrn,
            image,
            args.input_shape,
            args.output_shape,
            args.n_tiles,
            dataset["n_classes"]
        )

        output /= np.sum(output, axis=0, keepdims=True)

        nib.save(
            nib.Nifti1Image(np.float32(output).transpose(1, 2, 3, 0), affine),
            os.path.join(
                os.path.dirname(image_path),
                subject + args.output_suffix
            )
        )

if __name__ == '__main__':
    main()

ext: py

reason: import numpy


text:
#!/usr/bin/env python3
######################################################################
##        Copyright (c) 2021 Carsten Wulff Software, Norway
## ###################################################################
## Created       : wulff at 2021-8-10
## ###################################################################
##  The MIT License (MIT)
##
##  Permission is hereby granted, free of charge, to any person obtaining a copy
##  of this software and associated documentation files (the "Software"), to deal
##  in the Software without restriction, including without limitation the rights
##  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
##  copies of the Software, and to permit persons to whom the Software is
##  furnished to do so, subject to the following conditions:
##
##  The above copyright notice and this permission notice shall be included in all
##  copies or substantial portions of the Software.
##
##  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
##  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
##  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
##  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
##  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
##  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
##  SOFTWARE.
##
######################################################################

import os
import yaml
import re
import random
import numpy as np

runs=5
data = dict()
best = 1e6;
worst = 0;

random.seed(a=None)

os.system(" date > run.log && make clean 2>> run.log 1>> run.log")
for r in range(0,runs):
    seed = int(random.random()*np.power(2,17))
    status = os.system(f"make mazev solve SEED={seed} 2>> run.log 1>> run.log")

    if(status):
        os.system("echo \"Note: no $ allowed in verilog file\" >> run.log")
        exit(status)
    else:
        with open("result.yaml","r") as fi:
            obj = yaml.safe_load(fi)

        data[r] = obj

        if( obj["done_cycles"] > worst):
            worst = obj["done_cycles"]

        if( obj["done_cycles"] < best):
            best = obj["done_cycles"]

        with open("maze.txt","r") as fi:
            obj["maze"] = fi.read()

        with open("turtle.txt","r") as fi:
            obj["turtle"] = fi.read()

        with open("path.txt","r") as fi:
            obj["path"] = fi.read()

results = dict()
results["c_best"] = best
results["c_worst"] = worst
results["runs"] = runs
results["data"] = data

with open("allresults.yaml","w") as fo:
    fo.write(yaml.dump(results))

max_stars_repo_name: anderflu/dicex

max_issues_repo_name: anderflu/dicex

max_forks_repo_name: anderflu/dicex


text:
# Author: Christian Brodbeck <christianbrodbeck@nyu.edu>
"""Continuous neural data (CND) format used for mTRF-Toolbox"""
from pathlib import Path
from typing import Optional, Sequence, Union

import mne
from mne.externals.pymatreader import read_mat
import numpy

from .._data_obj import Dataset, Factor, NDVar, Scalar, Sensor, UTS, Var, combine, _matrix_graph
from .._io.fiff import mne_neighbor_files
from .._types import PathArg
from .._utils import ui

FILETYPES = [("CND files", "*.mat")]

def read_cnd(
        filename: PathArg = None,
        connectivity: Union[str, Sequence, float] = None,
) -> Optional[Dataset]:
    """Load continuous neural data (CND) file used in the mTRF-Toolbox

    Parameters
    ----------
    filename
        Path to the data file (``*.mat``). If unspecified, open a file dialog.
    connectivity
        Sensor adjacency graph for EEG sensors.
        By default, the function tries to use the ``deviceName`` entry and falls
        back on distance-based connectivity for unkown devices.
        Can be explicitly specified as a `FieldTrip neighbor file
        <https://www.fieldtriptoolbox.org/template/neighbours/>`_ (e.g., ``'biosemi64'``;
        Use a :class:`float` for distance-based connectivity (see :meth:`Sensor.set_connectivity`).
        For more options see :class:`Sensor`.

    Notes
    -----
    This format is experimental and the returned data format might change in the future;
    Please report any problems you might encounter.
    """
    if filename is None:
        path = ui.ask_file("Load CND File", "Select CND file to load as NDVar", FILETYPES)
        if not path:
            return
    else:
        path = Path(filename)
        if not path.suffix and not path.exists():
            path = path.with_suffix('.mat')
    data = read_mat(path)
    if 'eeg' in data:
        data_type = data['eeg']['dataType']
        # EEG sensor properties
        dist_connectivity = None
        sysname = data['eeg']['deviceName']
        ch_names = data['eeg']['chanlocs']['labels']
        # connectivity default
        if connectivity is None:
            available = mne_neighbor_files()
            if sysname in available:
                connectivity = sysname
            else:
                connectivity = 1.6
        # find connectivity
        if isinstance(connectivity, float):
            dist_connectivity = connectivity
            connectivity = 'none'
        elif connectivity is False:
            connectivity = 'none'
        elif isinstance(connectivity, str) and connectivity not in ('grid', 'none'):
            adj_matrix, adj_names = mne.channels.read_ch_adjacency(connectivity)
            # fix channel order
            if adj_names != ch_names:
                index = numpy.array([adj_names.index(name) for name in ch_names])
                adj_matrix = adj_matrix[index][:, index]
            connectivity = _matrix_graph(adj_matrix)
        locs = numpy.vstack([
            -numpy.array(data['eeg']['chanlocs']['Y']),
            data['eeg']['chanlocs']['X'],
            data['eeg']['chanlocs']['Z'],
        ]).T
        sensor = Sensor(locs, ch_names, sysname, connectivity=connectivity)
        if dist_connectivity:
            sensor.set_connectivity(connect_dist=dist_connectivity)
        # EEG data
        tstep = 1 / data['eeg']['fs']
        eeg = []
        for trial_data in data['eeg']['data']:
            uts = UTS(0, tstep, trial_data.shape[0])
            eeg.append(NDVar(trial_data, (uts, sensor), 'eeg'))
        # Trial position
        ds = Dataset({
            data_type.lower(): combine(eeg, to_list=True),
            'origTrialPosition': Var(data['eeg']['origTrialPosition'] - 1),
        })
        # Extra channels
        if 'extChan' in data['eeg']:
            desc = ds.info['extChan'] = data['eeg']['extChan']['description']
            extra_data = []
            for trial_data in data['eeg']['data']:
                uts = UTS(0, tstep, trial_data.shape[0])
                channel = Scalar('channel', range(trial_data.shape[1]))
                extra_data.append(NDVar(trial_data, (uts, channel), desc))
            ds['extChan'] = extra_data
        if 'reRef' in data['eeg']:
            ds.info['reRef'] = data['eeg']['reRef']
        return ds
    if 'stim' in data:
        ds = Dataset({'stimIdxs': Var(data['stim']['stimIdxs'] - 1)})
        # stim.data has size nStim x nRuns.
        tstep = 1 / data['stim']['fs']
        stim_names = data['stim']['names']
        for name, stim_data in zip(stim_names, data['stim']['data']):
            stim_ndvars = []
            for run_data in stim_data:
                uts = UTS(0, tstep, len(run_data))
                stim_ndvars.append(NDVar(run_data, uts, name))
            key = Dataset.as_key(name)
            ds[key] = combine(stim_ndvars, to_list=True)
        # Condition labels
        if 'condIdxs' in data['stim']:
            ds['condIdxs'] = Var(data['stim']['condIdxs'] - 1)
            labels = dict(enumerate(data['stim']['condNames'], 1))
            ds['condition'] = Factor(data['stim']['condIdxs'], labels=labels)
        return ds
    raise IOError("File contains neither 'eeg' or 'stim' entry")

max_stars_repo_name: christianbrodbeck/Eelbrain

max_issues_repo_name: christianbrodbeck/Eelbrain

max_forks_repo_name: christianbrodbeck/Eelbrain


text:
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the PyMVPA package for the
#   copyright and license terms.
#
### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Unit tests for rsa measures"""

from mvpa2.testing import *
skip_if_no_external('scipy')

from mvpa2.testing.datasets import datasets
from mvpa2.measures.anova import OneWayAnova

import numpy as np
from mvpa2.mappers.fx import *
from mvpa2.datasets.base import dataset_wizard, Dataset

from mvpa2.testing.tools import *

from mvpa2.measures.rsa import *
from mvpa2.base import externals
import scipy.stats as stats
from scipy.spatial.distance import pdist, squareform
from scipy.stats import rankdata, pearsonr

data = np.array([[ 0.22366105, 0.51562476, 0.62623543, 0.28081652, 0.56513533],
                [ 0.22077129, 0.63013374, 0.19641318, 0.38466208, 0.60788347],
                [ 0.64273055, 0.60455658, 0.71368501, 0.36652763, 0.51720253],
                [ 0.40148338, 0.34188668, 0.09174233, 0.33906488, 0.17804584],
                [ 0.60728718, 0.6110304 , 0.84817742, 0.33830628, 0.7123945 ],
                [ 0.32113428, 0.16916899, 0.53471886, 0.93321617, 0.22531679]])

def test_PDistConsistency():
    targets = np.tile(xrange(3),2)
    chunks = np.repeat(np.array((0,1)),3)
    # correct results
    cres1 = 0.41894348
    cres2 = np.array([[ 0.73062639, 0.16137995, 0.59441713]]).T
    dc1 = data[0:3,:] - np.mean(data[0:3,:],0)
    dc2 = data[3:6,:] - np.mean(data[3:6,:],0)
    center = squareform(np.corrcoef(pdist(dc1,'correlation'),pdist(dc2,'correlation')),
                        checks=False).reshape((1,-1))
    dsm1 = stats.rankdata(pdist(data[0:3,:],'correlation').reshape((1,-1)))
    dsm2 = stats.rankdata(pdist(data[3:6,:],'correlation').reshape((1,-1)))

    spearman = squareform(np.corrcoef(np.vstack((dsm1,dsm2))),
                        checks=False).reshape((1,-1))

    ds = dataset_wizard(samples=data, targets=targets, chunks=chunks)
    dscm = PDistConsistency()
    res1 = dscm(ds)
    dscm_c = PDistConsistency(center_data=True)
    res2 = dscm_c(ds)
    dscm_sp = PDistConsistency(consistency_metric='spearman')
    res3 = dscm_sp(ds)
    ds.append(ds)
    chunks = np.repeat(['one', 'two', 'three'], 4)
    ds.sa['chunks'] = chunks
    res4 = dscm(ds)
    dscm_sq = PDistConsistency(square=True)
    res4_sq = dscm_sq(ds)
    for i, p in enumerate(res4.sa.pairs):
        sqval =  np.asscalar(res4_sq[res4_sq.sa.chunks == p[0],
                                     res4_sq.sa.chunks == p[1]])
        assert_equal(sqval, res4.samples[i, 0])
    assert_almost_equal(np.mean(res1.samples),cres1)
    assert_array_almost_equal(res2.samples, center)
    assert_array_almost_equal(res3.samples, spearman)
    assert_array_almost_equal(res4.samples,cres2)

def test_PDist():
    targets = np.tile(xrange(3),2)
    chunks = np.repeat(np.array((0,1)),3)
    ds = dataset_wizard(samples=data, targets=targets, chunks=chunks)
    data_c = data - np.mean(data,0)
    # DSM matrix elements should come out as samples of one feature
    # to be in line with what e.g. a classifier returns -- facilitates
    # collection in a searchlight ...
    euc = pdist(data, 'euclidean')[None].T
    pear = pdist(data, 'correlation')[None].T
    city = pdist(data, 'cityblock')[None].T
    center_sq = squareform(pdist(data_c,'correlation'))

    # Now center each chunk separately
    dsm1 = PDist()
    dsm2 = PDist(pairwise_metric='euclidean')
    dsm3 = PDist(pairwise_metric='cityblock')
    dsm4 = PDist(center_data=True,square=True)
    assert_array_almost_equal(dsm1(ds).samples,pear)
    assert_array_almost_equal(dsm2(ds).samples,euc)
    dsm_res = dsm3(ds)
    assert_array_almost_equal(dsm_res.samples,city)
    # length correspondings to a single triangular matrix
    assert_equal(len(dsm_res.sa.pairs), len(ds) * (len(ds) - 1) / 2)
    # generate label pairs actually reflect the vectorform generated by
    # squareform()
    dsm_res_square = squareform(dsm_res.samples.T[0])
    for i, p in enumerate(dsm_res.sa.pairs):
        assert_equal(dsm_res_square[p[0], p[1]], dsm_res.samples[i, 0])
    dsm_res = dsm4(ds)
    assert_array_almost_equal(dsm_res.samples,center_sq)
    # sample attributes are carried over
    assert_almost_equal(ds.sa.targets, dsm_res.sa.targets)

def test_PDistTargetSimilarity():
    ds = Dataset(data)
    tdsm = range(15)
    ans1 = np.array([0.30956920104253222, 0.26152022709856804])
    ans2 = np.array([0.53882710751962437, 0.038217527859375197])
    ans3 = np.array([0.33571428571428574, 0.22121153763932569])
    tdcm1 = PDistTargetSimilarity(tdsm)
    tdcm2 = PDistTargetSimilarity(tdsm,
                                            pairwise_metric='euclidean')
    tdcm3 = PDistTargetSimilarity(tdsm,
                                comparison_metric = 'spearman')
    tdcm4 = PDistTargetSimilarity(tdsm,
                                    corrcoef_only=True)
    a1 = tdcm1(ds)
    a2 = tdcm2(ds)
    a3 = tdcm3(ds)
    a4 = tdcm4(ds)
    assert_array_almost_equal(a1.samples.squeeze(), ans1)
    assert_array_equal(a1.fa.metrics, ['rho', 'p'])
    assert_array_almost_equal(a2.samples.squeeze(), ans2)
    assert_array_equal(a2.fa.metrics, ['rho', 'p'])
    assert_array_almost_equal(a3.samples.squeeze(), ans3)
    assert_array_equal(a3.fa.metrics, ['rho', 'p'])
    assert_array_almost_equal(a4.samples.squeeze(), ans1[0])
    assert_array_equal(a4.fa.metrics, ['rho'])

def test_PDistTargetSimilaritySearchlight():
    # Test ability to use PDistTargetSimilarity in a searchlight
    from mvpa2.testing.datasets import datasets
    from mvpa2.mappers.fx import mean_group_sample
    from mvpa2.mappers.shape import TransposeMapper
    from mvpa2.measures.searchlight import sphere_searchlight
    ds = datasets['3dsmall'][:, :3]
    ds.fa['voxel_indices'] = ds.fa.myspace
    # use chunks values (4 of them) for targets
    ds.sa['targets'] = ds.sa.chunks
    ds = mean_group_sample(['chunks'])(ds)
    tdsm = np.arange(6)
    # We can run on full dataset
    tdcm1 = PDistTargetSimilarity(tdsm)
    a1 = tdcm1(ds)
    assert_array_equal(a1.fa.metrics, ['rho', 'p'])

    tdcm1_rho = PDistTargetSimilarity(tdsm, corrcoef_only=True)
    sl_rho = sphere_searchlight(tdcm1_rho)(ds)
    assert_array_equal(sl_rho.shape, (1, ds.nfeatures))

    # now with both but we need to transpose datasets
    tdcm1_both = PDistTargetSimilarity(tdsm, postproc=TransposeMapper())
    sl_both = sphere_searchlight(tdcm1_both)(ds)
    assert_array_equal(sl_both.shape, (2, ds.nfeatures))
    assert_array_equal(sl_both.sa.metrics, ['rho', 'p'])
    # rho must be exactly the same
    assert_array_equal(sl_both.samples[0], sl_rho.samples[0])
    # just because we are here and we can
    # Actually here for some reason assert_array_lequal gave me a trouble
    assert_true(np.all(sl_both.samples[1] <= 1.0))
    assert_true(np.all(0 <= sl_both.samples[1]))

def test_Regression():
    skip_if_no_external('skl')
    # a very correlated dataset
    corrdata = np.array([[1, 2], [10, 20], [-1, -2], [-10, -20]])
    # a perfect predictor
    perfect_pred = np.array([0, 2, 2, 2, 2, 0])

    ds = Dataset(corrdata)

    reg_types = ['lasso', 'ridge']

    # assert it pukes because predictor is not of the right shape
    assert_raises(ValueError, Regression, perfect_pred)

    # now make it right
    perfect_pred = np.atleast_2d(perfect_pred).T
    # assert it pukes for unknown method
    assert_raises(ValueError, Regression, perfect_pred, method='bzot')

    for reg_type in reg_types:
        regr = Regression(perfect_pred, alpha=0, fit_intercept=False,
                          rank_data=False, normalize=False, method=reg_type)
    coefs = regr(ds)
    assert_almost_equal(coefs.samples, 1.)

    # assert it pukes if predictor and ds have different shapes
    regr = Regression(perfect_pred)
    assert_raises(ValueError, regr, ds[:-1])

    # what if we select some items?
    keep_pairss = [range(3), [1], np.arange(3)]
    for reg_type in reg_types:
        for keep_pairs in keep_pairss:
            regr = Regression(perfect_pred, keep_pairs=keep_pairs, alpha=0,
                              fit_intercept=False, rank_data=False, normalize=False,
                              method=reg_type)
            coefs = regr(ds)
            assert_almost_equal(coefs.samples, 1.)

    # make a less perfect predictor
    bad_pred = np.ones((6, 1))
    predictors = np.hstack((perfect_pred, bad_pred))

    # check it works with combination of parameters
    from itertools import product
    outputs =  [np.array([[0.], [0.], [0.]]),
               np.array([[0.76665188], [0.], [0.]]),
               np.array([[ 0.5], [0.], [1.75]]),
               np.array([[0.92307692], [0.], [0.26923077]]),
               np.array([[0.], [0.], [ 3.70074342e-17]]),
               np.array([[8.57142857e-01], [0.], [-2.64338815e-17]]),
               np.array([[0.], [0.], [1.33333333]]),
               np.array([[0.84210526], [0.], [0.21052632]]),
               np.array([[0.], [0.]]),
               np.array([[0.76665188], [0.]]),
               np.array([[0.92982456], [0.]]),
               np.array([[0.92850288], [0.07053743]]),
               np.array([[0.], [0.]]),
               np.array([[0.85714286], [0.]]),
               np.array([[0.625], [0.]]),
               np.array([[0.87272727], [0.14545455]])]

    for i, (fit_intercept, rank_data, normalize, reg_type) in \
            enumerate(
                    product([True, False], [True, False],
                            [True, False], reg_types)):
        regr = Regression(predictors, alpha=1,
                               fit_intercept=fit_intercept, rank_data=rank_data,
                               normalize=normalize, method=reg_type)
        coefs = regr(ds)
        # check we get all the coefficients we need
        wanted_samples = 3 if fit_intercept else 2
        assert_equal(coefs.nsamples, wanted_samples)
        # check we get the actual output
        assert_almost_equal(coefs.samples, outputs[i])

max_stars_repo_name: effigies/PyMVPA

max_issues_repo_name: effigies/PyMVPA

max_forks_repo_name: effigies/PyMVPA

reason: import numpy,import scipy,from scipy


text:
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)
import sys

from .geom import geom
from ggplot.utils import is_categorical
import numpy as np

class geom_linerange(geom):
    """Plot intervals represented by vertical lines

    Parameters
    ---------

    x
        x values of data
    ymin
        lower end of the interval for each x
    ymax
        upper end of the interval for each x
    alpha : float
        alpha value, defaults to 1
    color : string
        line color, defaults to 'black'
    linetype : string
        line type, defaults to 'solid'
    size : string
        width of the line, defaults to 2

    Examples
    --------

    .. plot::
        :include-source:

        import numpy as np
        import pandas as pd
        from ggplot import *

        np.random.seed(42)
        x = np.linspace(0.5, 9.5, num=10)
        y = np.random.randn(10)
        ymin = y - np.random.uniform(0,1, size=10)
        ymax = y + np.random.uniform(0,1, size=10)

        data = pd.DataFrame({'x': x, 'ymin': ymin, 'ymax': ymax})

        ggplot(aes(x='x', ymin='ymin', ymax='ymax'), data) \
            + geom_linerange()

    """
    DEFAULT_AES = {'alpha': 1, 'color': 'black',
                   'linetype': 'solid',
                   'size': 2}
    REQUIRED_AES = {'x', 'ymin', 'ymax'}
    DEFAULT_PARAMS = {'stat': 'identity', 'position': 'identity', 'cmap': None}

    _aes_renames = {'size': 'linewidth', 'linetype': 'linestyle'}
    _units = {'alpha', 'color', 'linestyle'}

    def __init__(self, *args, **kwargs):
        super(geom_linerange, self).__init__(*args, **kwargs)
        self._warning_printed = False

    def _plot_unit(self, pinfo, ax):
        # If x is categorical, calculate positions to plot
        categorical = is_categorical(pinfo['x'])
        if categorical:
            x = pinfo.pop('x')
            new_x = np.arange(len(x))
            ax.set_xticks(new_x)
            ax.set_xticklabels(x)
            pinfo['x'] = new_x

        if 'linewidth' in pinfo and isinstance(pinfo['linewidth'], list):
            # ggplot also supports aes(size=...) but the current mathplotlib
            # is not. See https://github.com/matplotlib/matplotlib/issues/2658
            pinfo['linewidth'] = 4
            if not self._warning_printed:
                msg = "'geom_line()' currenty does not support the mapping of " +\
                      "size ('aes(size=<var>'), using size=4 as a replacement.\n" +\
                      "Use 'geom_line(size=x)' to set the size for the whole line.\n"
                sys.stderr.write(msg)
                self._warning_printed = True

        x = pinfo.pop('x')
        x = np.vstack([x, x])

        ymin = pinfo.pop('ymin')
        ymax = pinfo.pop('ymax')
        y = np.vstack([ymin, ymax])

        ax.plot(x, y, **pinfo)

max_stars_repo_name: ouseful-backup/ggplot

max_issues_repo_name: vannem/ggplot

max_forks_repo_name: vannem/ggplot


text:
from CCH import CCH
from config import *
from GSM import *
import numpy as np

class BCCH(CCH):

	def __init__(self,slot):
		CCH.__init__(self)
		self.config = (range(2,6),slot)
		self.name = "BCCH"

	def callback(self,b,fn,state):
		b.training = state.bcc
		return CCH.callback(self,b,fn,state)


text:
#!/usr/bin/env python
""" LIP control """

import rospy
import math
from std_msgs.msg import Float64
from sensor_msgs.msg import JointState
from geometry_msgs.msg import Vector3, WrenchStamped
import numpy as np

class LIPController(object):
    """ LIP Control """
    def __init__(self, timestep):
        self.velocity = 0
        self.effort = 0
        self.position = 0
        self.kp = 150
        self.ki = 10
        self.kd = 150
        self.target_theta = 0
        self.smp_time = 0.01
        self.theta0 = 0
        self.thetai = 0
        self.error = 0
        self.duty_ratio = 0
        self.pubs = rospy.Publisher("/lip/joint_effort_controller_j_0/command/", Float64, queue_size=10)
        self.command_sub = rospy.Subscriber("/lip/joint_states", JointState , self.callback)

    def callback(self, data):
        """ Callback function for subscribing /lip/joint_states topic
        Use this data in your control equation below """
        self.effort = data.effort
        self.velocity = data.velocity
        self.position = data.position
        """ Write your control equation in self.cmd
        to publish the effort values to the Joint Effort Controller """
        self.error = self.target_theta - self.position[0]
       	self.dtheta = (self.error - self.theta0)/self.smp_time
        self.theta0 = self.error
        self.thetai += self.error*self.smp_time
        self.duty_ratio = (self.error*self.kp) + (self.thetai*self.ki) + (self.dtheta*self.kd)
        self.cmd = self.duty_ratio
        if self.cmd > 100:
        	self.cmd = 100
        elif self.cmd < -100:
        	self.cmd = -100
        self.pubs.publish(self.cmd)

def main():
    rospy.init_node('lip_control', anonymous=False)
    controller = LIPController(.01)
    rospy.spin()
    del controller
    return

if __name__ == "__main__":
    try:
        main()
    except rospy.ROSInterruptException:
        pass


text:
#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import logging
from paddle.fluid.op import Operator, DynamicRecurrentOp
import paddle.fluid.core as core
import unittest
import numpy as np
import paddle.fluid as fluid
from paddle.fluid.framework import Program, program_guard

def create_tensor(scope, name, np_data):
    tensor = scope.var(name).get_tensor()
    tensor.set(np_data, core.CPUPlace())
    return tensor

class BeamSearchOpTester(unittest.TestCase):
    """unittest of beam_search_op"""

    def setUp(self):
        self.scope = core.Scope()
        self._create_ids()
        self._create_pre_scores()
        self._create_scores()
        self._create_pre_ids()
        self.scope.var('selected_ids')
        self.scope.var('selected_scores')
        self.scope.var('parent_idx')

    def test_run(self):
        op = Operator(
            'beam_search',
            pre_ids='pre_ids',
            pre_scores='pre_scores',
            ids='ids',
            scores='scores',
            selected_ids='selected_ids',
            selected_scores='selected_scores',
            parent_idx='parent_idx',
            level=0,
            beam_size=2,
            end_id=0, )
        op.run(self.scope, core.CPUPlace())
        selected_ids = self.scope.find_var("selected_ids").get_tensor()
        selected_scores = self.scope.find_var("selected_scores").get_tensor()
        parent_idx = self.scope.find_var("parent_idx").get_tensor()
        self.assertTrue(
            np.allclose(
                np.array(selected_ids), np.array([4, 2, 3, 8])[:, np.newaxis]))
        self.assertTrue(
            np.allclose(
                np.array(selected_scores),
                np.array([0.5, 0.6, 0.9, 0.7])[:, np.newaxis]))
        self.assertEqual(selected_ids.lod(), [[0, 2, 4], [0, 1, 2, 3, 4]])
        self.assertTrue(
            np.allclose(np.array(parent_idx), np.array([0, 1, 2, 3])))

    def _create_pre_ids(self):
        np_data = np.array([[1, 2, 3, 4]], dtype='int64')
        tensor = create_tensor(self.scope, 'pre_ids', np_data)

    def _create_pre_scores(self):
        np_data = np.array([[0.1, 0.2, 0.3, 0.4]], dtype='float32')
        tensor = create_tensor(self.scope, 'pre_scores', np_data)

    def _create_ids(self):
        self.lod = [[0, 2, 4], [0, 1, 2, 3, 4]]
        np_data = np.array(
            [[4, 2, 5], [2, 1, 3], [3, 5, 2], [8, 2, 1]], dtype='int64')
        tensor = create_tensor(self.scope, "ids", np_data)
        tensor.set_lod(self.lod)

    def _create_scores(self):
        np_data = np.array(
            [
                [0.5, 0.3, 0.2],
                [0.6, 0.3, 0.1],
                [0.9, 0.5, 0.1],
                [0.7, 0.5, 0.1],
            ],
            dtype='float32')
        tensor = create_tensor(self.scope, "scores", np_data)
        tensor.set_lod(self.lod)

class TestBeamSearchOpError(unittest.TestCase):
    def test_errors(self):
        with program_guard(Program(), Program()):
            pre_ids = fluid.layers.data(
                name='pre_id', shape=[1], lod_level=2, dtype='int64')
            pre_scores = fluid.layers.data(
                name='pre_scores', shape=[1], lod_level=2, dtype='float32')
            probs = fluid.layers.data(
                name='probs', shape=[10000], dtype='float32')
            topk_scores, topk_indices = fluid.layers.topk(probs, k=4)
            accu_scores = fluid.layers.elementwise_add(
                x=fluid.layers.log(x=topk_scores),
                y=fluid.layers.reshape(
                    pre_scores, shape=[-1]),
                axis=0)

            def test_preids_Variable():
                # the input pre_ids must be Variable
                input_data = np.random.randint(1, 5, [5, 1]).astype("int64")
                fluid.layers.beam_search(
                    pre_ids=input_data,
                    pre_scores=pre_scores,
                    ids=topk_indices,
                    scores=accu_scores,
                    beam_size=4,
                    end_id=1)

            self.assertRaises(TypeError, test_preids_Variable)

            def test_prescores_Variable():
                # the input pre_scores must be Variable
                input_data = np.random.uniform(1, 5, [5, 1]).astype("float32")
                fluid.layers.beam_search(
                    pre_ids=pre_ids,
                    pre_scores=input_data,
                    ids=topk_indices,
                    scores=accu_scores,
                    beam_size=4,
                    end_id=1)

            self.assertRaises(TypeError, test_prescores_Variable)

            def test_ids_Variable():
                # the input ids must be Variable or None
                input_data = np.random.randint(1, 5, [5, 1]).astype("int64")
                fluid.layers.beam_search(
                    pre_ids=pre_ids,
                    pre_scores=pre_scores,
                    ids=input_data,
                    scores=accu_scores,
                    beam_size=4,
                    end_id=1)

            self.assertRaises(TypeError, test_ids_Variable)

            def test_scores_Variable():
                # the input scores must be Variable
                input_data = np.random.randint(1, 5, [5, 1]).astype("float32")
                fluid.layers.beam_search(
                    pre_ids=pre_ids,
                    pre_scores=pre_scores,
                    ids=topk_indices,
                    scores=input_data,
                    beam_size=4,
                    end_id=1)

            self.assertRaises(TypeError, test_scores_Variable)

            def test_preids_dtype():
                # the dtype of input pre_ids must be int64
                input_data = fluid.layers.data(
                    name='input_data', shape=[1], lod_level=2, dtype='float32')
                fluid.layers.beam_search(
                    pre_ids=input_data,
                    pre_scores=pre_scores,
                    ids=topk_indices,
                    scores=accu_scores,
                    beam_size=4,
                    end_id=1)

            self.assertRaises(TypeError, test_preids_dtype)

            def test_prescores_dtype():
                # the dtype of input pre_scores must be float32
                input_data = fluid.layers.data(
                    name='input_data', shape=[1], lod_level=2, dtype='int64')
                fluid.layers.beam_search(
                    pre_ids=pre_ids,
                    pre_scores=input_data,
                    ids=topk_indices,
                    scores=accu_scores,
                    beam_size=4,
                    end_id=1)

            self.assertRaises(TypeError, test_prescores_dtype)

if __name__ == '__main__':
    unittest.main()

max_stars_repo_name: jinyuKING/Paddle

max_issues_repo_name: jinyuKING/Paddle

max_forks_repo_name: jinyuKING/Paddle


text:
import os
import sys
from tqdm import tqdm
import numpy as np

fish_label ={"ALB":0, "BET":1, "DOL":2, "LAG":3,"SHARK":4, "YFT":5, "NoF":6, "OTHER":7}

def split_train_val():
    np.random.seed(100)

    ALL_PATH = "../data/all.txt"
    TRAIN_PATH = "../data/train.txt"
    VAL_PATH = "../data/val.txt"

    with open(ALL_PATH, "r") as f:
        all_data = f.readlines()

    f_train = open(TRAIN_PATH, "w")
    f_val = open(VAL_PATH, "w")

    n_data = len(all_data)
    print("Number of data: {}".format(n_data))

    ratio_val = 0.15
    n_val = int(n_data * ratio_val)
    n_train = n_data - n_val

    assert n_train + n_val == n_data

    print("Number of train data: {}".format(n_train))
    print("NUmber of validation data: {}".format(n_val))

    np.random.shuffle(all_data)

    for i, line in enumerate(all_data):
        if i < n_train:
            f_train.write(line)
        else:
            f_val.write(line)

    f_train.close()
    f_val.close()

def make_list():
    TRAIN_PATH = "/media/matterd/0C228E28228E173C/dataset/kaggle/fisheries_monitoring/train/train/"
    #VAL_PATH = "/media/matterd/0C228E28228E173C/dataset/kaggle/fisheries_monitoring/val/"
    OUT_PATH = "../data/all.txt"

    # for all data
    folders = os.listdir(TRAIN_PATH)
    print(folders)
    #folders = [x + TRAIN_PATH for x in temp_folders]

    f = open(OUT_PATH, "w")

    for folder in tqdm(folders):
        if os.path.isdir(TRAIN_PATH + folder) == False:
            continue

        files = os.listdir(TRAIN_PATH + folder)
        for file in files:
            f.write(folder + "/" + file)
            f.write(" ")
            f.write(str(fish_label[folder]))
            f.write("\n")

    f.close()

if __name__ == "__main__":
    make_list()
    split_train_val()


text:
import os

import numpy as np
import torch

from .metric_computer import MetricComputer
from ..common_util.image import get_gt_frame, get_comp_frame
from ..models.i3d.pytorch_i3d import InceptionI3d
from ..fid import calculate_frechet_distance
from ..fid.util import extract_video_features

SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
PROJ_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, '..', '..'))

class VFIDComputer(MetricComputer):

    def compute_metric(self):
        num_work_items = 2 * len(self.opts.video_names) if self.opts.gt_vfid_feats_path is None \
            else len(self.opts.video_names)
        self.send_work_count_msg(num_work_items)

        model = InceptionI3d(400, in_channels=3)
        weights = torch.load(os.path.join(PROJ_DIR, 'pretrained_models', 'rgb_imagenet.pt'))
        model.load_state_dict(weights)
        model.cuda()
        model.eval()
        torch.set_grad_enabled(False)

        def get_gt_frame_wrapper(video_name, t):
            return get_gt_frame(self.opts.gt_root, video_name, t)

        def get_comp_frame_wrapper(video_name, t):
            return get_comp_frame(self.opts.gt_root, self.opts.pred_root, video_name, t)

        def update_progress_cb():
            self.send_update_msg(1)

        if self.opts.gt_vfid_clips_feats_path is None:
            gt_clip_features = extract_video_features(model, get_gt_frame_wrapper, self.opts.video_names,
                                                      self.opts.video_frame_counts, update_progress_cb)
        else:
            gt_clip_features = np.load(self.opts.gt_vfid_feats_path)
        gt_mu = np.mean(gt_clip_features, axis=0)
        gt_sigma = np.cov(gt_clip_features, rowvar=False)

        comp_clip_feats = extract_video_features(model, get_comp_frame_wrapper, self.opts.video_names,
                                                 self.opts.video_frame_counts, update_progress_cb)
        comp_mu = np.mean(comp_clip_feats, axis=0)
        comp_sigma = np.cov(comp_clip_feats, rowvar=False)

        vfid_all = calculate_frechet_distance(gt_mu, gt_sigma, comp_mu, comp_sigma)

        self.send_result_msg(vfid_all)

max_stars_repo_name: MichiganCOG/video-inpainting-evaluation

max_issues_repo_name: MichiganCOG/video-inpainting-evaluation

max_forks_repo_name: MichiganCOG/video-inpainting-evaluation


text:
import numpy
from audioread import NoBackendError
import pandas
import sys
MODULE_PATH = '/home/macbookretina/automatic-music-genre-classification/feature_extraction_deep_learning'
sys.path.insert(1, MODULE_PATH)
from custom_module.utilities import *

# def process_and_save(cqts, mel_spects, genre_labels, count):
#     # check if all the lists are equal in length and throw an exception if not
#     path = '/home/macbookretina/s3-bucket/data'
#     count = str(count)
#     print('checking if the lists are equal in size.')
#     is_all_equal_in_length = len(cqts) == len(mel_spects) == len(genre_labels)
#     assert (is_all_equal_in_length), \
#                 'cqts: ' + str(len(cqts)) + \
#                 ' mel_spects: ' + str(len(mel_spects)) + \
#                 ' genre_labels: ' + str(len(genre_labels))

#     # convert the lists to arrays so it can be stacked
#     print('converting lists to numpy array')
#     cqts = numpy.array(cqts)
#     mel_spects = numpy.array(mel_spects)
#     length = len(cqts)
#     genre_labels = numpy.array(genre_labels).reshape(length, 1)
#     data_sources = numpy.array(['fma']*length).reshape(length, 1)
#     data_cqts = {
#         'genre_labels': genre_labels,
#         'data_sources': data_sources,
#         'cqts': cqts
#     }
#     data_mel = {
#         'genre_labels': genre_labels,
#         'data_sources': data_sources,
#         'cqts': cqts
#     }

#     # create dataframes and save as csvs
#     print('stacking array & saving as csv')
# #     cqt_df = pandas.DataFrame(numpy.hstack((genre_labels, data_sources, cqts)))
# #     mel_spect_df = pandas.DataFrame(numpy.hstack((genre_labels, data_sources, mel_spects)))
#     cqt_df = pandas.DataFrame(data_cqts)
#     mel_spect_df = pandas.DataFrame(data_mel)
#     cqt_df.to_csv(path + '/cqt_fma_' + count + '.csv')
#     mel_spect_df.to_csv(path + '/mel_spect_fma_' + count + '.csv')
#     print('saved batch: ' + count + '!!')
#     return [], [], []

def extract_from_fma():
    # extract log-mel / constant-Q transform and mel-spectomgram in fma

    # collect track id and genres of tracks in the small subset.
    print('collecting track id and genres of tracks in the small subset of fma dataset')
    tracks = load(MOUNTED_DATASET_PATH + '/fma_metadata/tracks.csv')
    fma_full = tracks[[('set', 'subset'), ('track', 'genre_top')]]
    small_subset = fma_full[('set', 'subset')] == 'small'
    fma_small = fma_full[small_subset]
    fma_small = pd.DataFrame({
        'subset': fma_small[('set', 'subset')],
        'label': fma_small[('track', 'genre_top')]
    })
    print('collected track id and genres of tracks in the small subset of fma')

    # create an empty list to store extract feature and label
    cqts = []
    mel_spects = []
    genre_labels = []
    data_sources = []
    print('extracting log-mel and mel-spectogram from fma dataset')
    count = 0
    for directory in os.scandir(MOUNTED_DATASET_PATH + '/fma_small'):
        if directory.is_dir():
            for file in os.scandir(directory.path):
#                 if count == 1:
#                     cqts, mel_spects, genre_labels = process_and_save(
#                         cqts, mel_spects, genre_labels, count)

                if file.is_file():
                    # extract track id and map track id to genre label
                    track_id = int(file.name[:-4].lstrip('0'))
                    genre_label = fma_small.at[track_id, 'label'].lower().replace('-', '')

                    if genre_label in GENRES:
                        try:
                            scaled_cqt = extract_cqt(file);
                            scaled_mel_spect = extract_mel_spect(file);
                        except RuntimeError:
                            print('RuntimeError')
                            continue
                        except NoBackendError:
                            print('NoBackendError')
                            continue

                        # adjust shape to the shape with most occurence
                        if scaled_cqt.shape[1] != 2812:
                            scaled_cqt.resize(84, 2812, refcheck=False)

                        if scaled_mel_spect.shape[1] != 2812:
                            scaled_mel_spect.resize(128, 2812, refcheck=False)

                        # append to list
                        genre_labels.append(genre_label)
                        # flatten to fit into dataframe and add to the list
                        scaled_cqt = scaled_cqt.flatten()
                        cqts.append(scaled_cqt)
                        scaled_mel_spect = scaled_mel_spect.flatten()
                        mel_spects.append(scaled_mel_spect)

                        feedback(file, genre_label)
            count = count + 1

    # check if all the lists are equal in length and throw an exception if not
    print('checking if the lists are equal in size.')
    is_all_equal_in_length = len(cqts) == len(mel_spects) == len(genre_labels)
    assert (is_all_equal_in_length), \
                'cqts: ' + str(len(cqts)) + \
                ' mel_spects: ' + str(len(mel_spects)) + \
                ' genre_labels: ' + str(len(genre_labels))

    # convert the lists to arrays so it can be stacked
    print('converting lists to numpy array')
    cqts = numpy.array(cqts)
    mel_spects = numpy.array(mel_spects)
    length = len(cqts)
    genre_labels = numpy.array(genre_labels).reshape(length, 1)
    data_sources = numpy.array(['fma']*length).reshape(length, 1)

#     # create dataframes and save as csvs
#     print('stacking array & saving as csv')
#     cqt_df = pandas.DataFrame(numpy.hstack((genre_labels, data_sources, cqts)))
#     mel_spect_df = pandas.DataFrame(numpy.hstack((genre_labels, data_sources, mel_spects)))
#     cqt_df.to_csv(path + '/cqt_fma.csv')
#     mel_spect_df.to_csv(path + '/mel_spect_fma.csv')
#     print('done')

if __name__ == '__main__':
    extract_from_fma()

max_stars_repo_name: DKMDebugin/amgc

max_issues_repo_name: DKMDebugin/amgc

max_forks_repo_name: DKMDebugin/amgc


text:
# -*- coding: utf-8 -*-
import os
import src.Constant as Constant
from src.face_landmark import FaceMarks as Landmark
import cv2
from scipy import misc

DATASET_NAME = "lfw"
destDir = os.path.join(Constant.ROOT_PATH, "datasets",DATASET_NAME + "_trans")
DATASET = os.path.join(Constant.ROOT_PATH, "datasets", DATASET_NAME)
FRONT_DATA = os.path.join(Constant.ROOT_PATH, "datasets", DATASET_NAME + "_front")
landmark = Landmark(faceArea=Constant.FACE_AREA)
OUT_SIZE = 256
REPLACE_OLD = False

if not os.path.isdir(destDir):
    os.mkdir(destDir)
if not os.path.isdir(FRONT_DATA):
    os.mkdir(FRONT_DATA)

totalLength = str(len(os.listdir(DATASET)))
for index, subFolder in enumerate(os.listdir(DATASET)):
    print("processing " + subFolder + " " + str(index+1) + " of total " + totalLength)
    subFolderPath = os.path.join(DATASET, subFolder)
    totalSubFile = str(len(os.listdir(subFolderPath)))
    for i, imgFile in enumerate(os.listdir(subFolderPath)):
        print("processing " + imgFile + " " + str(i+1) + " of total " + totalSubFile
                    + " in subFolder " + subFolder + " " + str(index+1) + "/" + str(totalLength))
        if not REPLACE_OLD and os.path.exists(os.path.join(destDir, imgFile)):
            continue
        imgPath = os.path.join(subFolderPath, imgFile)
        img = cv2.cvtColor(cv2.imread(imgPath), cv2.COLOR_BGR2RGB)
        frontImg = landmark.copyFront(img)
        if frontImg is None:
            transImg = landmark.landmark_transform(img)
            if transImg is None:
                print("transform failed: file %s has no face" % imgFile)
                continue
            resizeImg = misc.imresize(transImg, (OUT_SIZE, OUT_SIZE))
            misc.imsave(os.path.join(destDir, imgFile), resizeImg)
        else:
            resizeImg = misc.imresize(frontImg, (OUT_SIZE, OUT_SIZE))
            misc.imsave(os.path.join(FRONT_DATA, imgFile), resizeImg)

max_stars_repo_name: corganhejijun/frontal-trans

max_issues_repo_name: corganhejijun/frontal-trans

max_forks_repo_name: corganhejijun/frontal-trans

reason: from scipy


text:
import sys
import gzip
import cPickle
import tensorflow as tf
import numpy as np
import time
import math
import os
import matplotlib.pyplot as plt
import pylab
from deepgmap.post_train_tools import unpooling
import importlib as il
import getopt

def test_batch(test_batch_file):
    with np.load(test_batch_file) as f:
        dnase_data_labels1=f['labels'], f['data_array']
        images=np.reshape(dnase_data_labels1[1], (batch_size, data_length, 4, 1))
        labels=dnase_data_labels1[0]
    return images, labels

start=time.time()

try:
    options, args =getopt.getopt(sys.argv[1:], 'm:t:n:o:d:', ['model=','test_batch=','network_constructor=','output_dir=','deconv='])
except getopt.GetoptError as err:
    #print str(err)
    sys.exit(2)
if len(options)<3:
    print('too few argument')
    sys.exit(0)
for opt, arg in options:
    if opt in ('-m', '--model'):
        trained_model=arg
    elif opt in ('-t', '--test_batch'):
        test_batch_file=arg
    elif opt in ('-n', '--network_constructor'):
        network_constructor=arg
    elif opt in ('-o', '--output_dir'):
        output_dir=arg
    elif opt in ('-d','--deconv'):
        deconv=arg

#output_dir=None

config = tf.ConfigProto()
config.gpu_options.allow_growth=True
sess = tf.Session(config=config)

keep_prob = tf.placeholder(tf.float32)
keep_prob2 = tf.placeholder(tf.float32)
keep_prob3 = tf.placeholder(tf.float32)

x_image = tf.placeholder(tf.float32, shape=[None, 1000, 4, 1])
y_ = tf.placeholder(tf.float32, shape=[None, 20])
phase=tf.placeholder(tf.bool)
dropout_1=0.95
dropout_2=0.9
dropout_3=0.85
batch_size=100
data_length=1000
input_dir=trained_model
nc=il.import_module("network_constructors."+str(network_constructor))
train_speed=0.00005
a=time.asctime()
b=a.replace(':', '')
start_at=b.replace(' ', '_')

model = nc.Model(image=x_image, label=y_,
                 output_dir=output_dir,
                 phase=phase,
                 start_at=start_at,
                 keep_prob=keep_prob,
                 keep_prob2=keep_prob2,
                 keep_prob3=keep_prob3,
                 data_length=data_length)

sess.run(tf.global_variables_initializer())
saver=model.saver
saver.restore(sess, input_dir)

batch = test_batch(test_batch_file)
test_accuracy1, y_label1, y_prediction1 =sess.run([model.error, y_, model.prediction[1]], feed_dict={x_image: batch[0], y_: batch[1], keep_prob: 1.0, keep_prob2: 1.0, keep_prob3: 1.0})
#print "test accuracy (true:false=5:5): "+str(test_accuracy1)
#print deconv

if deconv=="transpose":

    def conv2d_tp(x, W, output_shape):
        return tf.nn.conv2d_transpose(x, W, output_shape, strides=[1, 1, 1, 1], padding='VALID')
    def conv2d_tp2(x, W, output_shape):
        return tf.nn.conv2d_transpose(x, W, output_shape, strides=[1,2, 1,1], padding='VALID')
    def conv2d_tp4(x, W, output_shape):
        return tf.nn.conv2d_transpose(x, W, output_shape, strides=[1, 4, 1, 1], padding='VALID')
    def max_pool_2x1(x):
        return tf.nn.max_pool_with_argmax(x, ksize=[1, 2, 1, 1], strides=[1, 2, 1, 1], padding='SAME')
    def max_pool_4x1(x):
        return tf.nn.max_pool_with_argmax(x, ksize=[1, 4, 1, 1], strides=[1, 4, 1, 1], padding='SAME')

    index_of_image=0
    positive_image=[]
    for y in batch[1]:
        #print y[0]
        if np.sum(y)>0:
            positive_image.append(index_of_image)
        index_of_image+=1
    #print len(positive_image)
    for k in range(len(positive_image)):

        images4=np.reshape(batch[0][positive_image[k]], (1, data_length, 4, 1))

        #h_conv3_, h_conv25_, h_conv24_, h_conv23_, h_conv22_,h_conv21_, h_conv2_, h_conv1_, b_conv3_=sess.run([h_conv3, h_conv25,h_conv24, h_conv23, h_conv22,h_conv21, h_conv2, h_conv1, b_conv3],

        #                                                                                feed_dict={x_image: images4, keep_prob: 1.0, keep_prob2: 1.0, keep_prob3: 1.0})
        y,y_sgm, variables_dict, neurons_dict, _3=sess.run(model.prediction, feed_dict={x_image: images4, keep_prob: 1.0, keep_prob2: 1.0, keep_prob3: 1.0, phase: False})
        """
            variable_dict={"W_conv1": W_conv1,
                           "W_conv2": W_conv2,
                           "W_conv21": W_conv21,
                           "W_conv22": W_conv22,
                           "W_fc1": W_fc1,
                           "W_fc4": W_fc4,
                           "b_fc1": b_fc1,
                           "b_fc4": b_fc4}
            neurons_dict={"h_conv22":h_conv22,
                          "h_conv21":h_conv21,
                          "h_conv2":h_conv2,
                          "h_conv11":h_conv11,
                          "h_conv12":h_conv12,
                          "h_fc1_drop": h_fc1_drop,
                          "h_pool3_flat":h_pool3_flat,
                          "h_pool22":h_pool22,
                          "h_pool21":h_pool21,
                          "h_pool2":h_pool2,
                          "h_pool1":h_pool1,
                          "h_pool1_rc":h_pool1_rc}
        """

        h_conv11_,\
        h_conv12_,\
        h_conv2_,\
        h_conv21_,\
        h_conv22_,\
        h_pool1_,\
        h_pool1_rc_,\
        h_pool2_,\
        h_pool21_,\
        h_pool22_ =\
        neurons_dict["h_conv11"],\
        neurons_dict["h_conv12"],\
        neurons_dict["h_conv2"],\
        neurons_dict["h_conv21"],\
        neurons_dict["h_conv22"],\
        neurons_dict["h_pool1"],\
        neurons_dict["h_pool1_rc"],\
        neurons_dict["h_pool2"],\
        neurons_dict["h_pool21"],\
        neurons_dict["h_pool22"]

        sess2 = tf.Session()
        #print h_pool21_
        h_pool21_shape=list(h_pool21_.shape)
        h_pool21_t4= conv2d_tp(h_conv22_, variables_dict["W_conv22"], h_pool21_shape)
        _, mask21=max_pool_2x1(h_conv21_)
        #h_unpool21_t4=unpooling.unpool(h_pool21_t4, mask21,output_shape=h_conv21_.shape)
        h_unpool21_t4=unpooling.unpool2(h_pool21_t4, mask21)

        h_pool2_shape=list(h_pool2_.shape)
        h_pool2_t4= conv2d_tp(h_unpool21_t4, variables_dict["W_conv21"], h_pool2_shape)
        _, mask2=max_pool_2x1(h_conv2_)
        #h_unpool2_t4=unpooling.unpool(h_pool2_t4,mask2,output_shape=h_conv2_.shape)
        h_unpool2_t4=unpooling.unpool2(h_pool2_t4,mask2)

        h_pool1_shape=list(h_pool1_.shape)
        h_pool1_t4= conv2d_tp(h_unpool2_t4, variables_dict["W_conv2"], h_pool1_shape)
        _,mask1=max_pool_2x1(h_conv11_)
        #h_unpool1_t4=unpooling.unpool(h_pool1_t4,mask1,output_shape=h_conv11_.shape)
        h_unpool1_t4=unpooling.unpool2(h_pool1_t4,mask1)

        h_pool1_rc_t4=conv2d_tp(h_unpool2_t4, tf.reverse(variables_dict["W_conv2"], [0,1]), h_pool1_shape)
        _,mask1rc=max_pool_2x1(h_conv12_)
        #h_unpool1_rc_t4=unpooling.unpool(h_pool1_rc_t4,mask1rc,output_shape=h_conv12_.shape)
        h_unpool1_rc_t4=unpooling.unpool2(h_pool1_rc_t4,mask1rc)

        reconstruction_shape=images4.shape
        #print reconstruction_shape
        reconstruction_conv22=conv2d_tp(h_unpool1_t4, variables_dict["W_conv1"], reconstruction_shape)+conv2d_tp(h_unpool1_rc_t4, tf.reverse(variables_dict["W_conv1"], [0,1]), reconstruction_shape)

        sess2.run(tf.global_variables_initializer())
        units_conv22 = sess2.run(reconstruction_conv22)

        reshaped_conv22=np.reshape(units_conv22, (data_length, 4))

        # Compute and plot first dendrogram.
        fig = plt.figure(figsize=(12,8))

        # Plot distance matrix.

        axmatrix_conv22 = fig.add_axes([0.05,0.05,0.1,0.9])
        im_conv22 = axmatrix_conv22.matshow(reshaped_conv22, aspect='auto', origin='lower', cmap=plt.get_cmap('YlGnBu'))
        axmatrix_conv22.set_xticks([])
        axmatrix_conv22.set_yticks([])
        axcolor = fig.add_axes([0.16,0.05,0.02,0.9])
        pylab.colorbar(im_conv22, cax=axcolor)

        reshaped2=np.reshape(images4, (data_length, 4))
        axmatrix3 = fig.add_axes([0.85,0.05,0.1,0.9])
        im3 = axmatrix3.matshow(reshaped2, aspect='auto', origin='lower', cmap=plt.get_cmap('YlGnBu'))
        axmatrix3.set_xticks([])
        axmatrix3.set_yticks([])
        axcolor = fig.add_axes([0.96,0.05,0.02,0.9])
        pylab.colorbar(im3, cax=axcolor)

        np.savez_compressed(str(output_dir)+str(trained_model.split('/')[-1])+"_transpose_"+str(k),
                            conv22=reshaped_conv22,
                            original=np.reshape(images4,(data_length, 4)))

        fig.savefig(str(output_dir)+str(trained_model.split('/')[-1])+'_reconstruction_'+str(k)+'.png')
        #plt.show()
        sess2.close()
    sess.close()
elif deconv=="train":
    def recon_variable(shape, variable_name):
        initial = tf.truncated_normal(shape, mean=0.50, stddev=0.5)
        return tf.Variable(initial, name=variable_name, trainable=True)
    def conv2d_1(x, W):
            return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='VALID')
    def conv2d(x, W):
        return tf.nn.conv2d(x, W, strides=[1, 2, 1, 1], padding='VALID')

    def max_pool_2x2(x):
        return tf.nn.max_pool(x, ksize=[1, 2, 1, 1], strides=[1, 2, 1, 1], padding='SAME')
    def max_pool_4x1(x):
        return tf.nn.max_pool(x, ksize=[1, 4, 1, 1], strides=[1, 4, 1, 1], padding='SAME')
    def max_pool_2x2_with_argmax(x):
        return tf.nn.max_pool_with_argmax(x, ksize=[1, 2, 1, 1], strides=[1, 2, 1, 1], padding='SAME')
    def max_pool_4x1_with_argmax(x):
        return tf.nn.max_pool_with_argmax(x, ksize=[1, 4, 1, 1], strides=[1, 4, 1, 1], padding='SAME')

    index_of_image=0
    positive_image=[]
    """for y in batch[1]:
        print y[0]
        if y[0]==1:
            positive_image.append(np.reshape(batch[0][index_of_image], [1,1000,4,1]))
        index_of_image+=1"""

    #print positive_image[0]

    _1, _2, variavl_dict, neurons_dict,_3=sess.run(model.prediction, feed_dict={x_image: batch[0], keep_prob: 1.0, keep_prob2: 1.0, keep_prob3: 1.0})
    fc1_param=model.fc1_param
    dimension22=model.dimension22

    sess.close()

    sess2 = tf.Session()

    x_image_recon = recon_variable([1, 1000, 4, 1], 'x_image_recon')

    h_conv11_re=conv2d_1(x_image_recon, variavl_dict["W_conv1"])
    h_conv12_re=conv2d_1(x_image_recon, tf.reverse(variavl_dict["W_conv1"], [0,1]))
    h_conv11_re_ = tf.nn.relu(h_conv11_re)
    h_conv12_re_ = tf.nn.relu(h_conv12_re)
    h_pool1_re = max_pool_2x2(h_conv11_re_)
    h_pool1_rc_re = max_pool_2x2(h_conv12_re_)
    h_conv2_re = tf.add(tf.nn.relu(conv2d_1(h_pool1_re, variavl_dict["W_conv2"])), tf.nn.relu(conv2d_1(h_pool1_rc_re, tf.reverse(variavl_dict["W_conv2"], [0,1]))))
    h_pool2_re = max_pool_2x2(h_conv2_re)
    h_conv21_re = tf.nn.relu(conv2d_1(h_pool2_re, variavl_dict["W_conv21"]))
    h_pool21_re = max_pool_2x2(h_conv21_re)
    h_conv22_re = tf.nn.relu(conv2d_1(h_pool21_re, variavl_dict["W_conv22"]))
    h_pool22_re = max_pool_4x1(h_conv22_re)

    h_pool3_flat_re = tf.reshape(h_pool22_re, [-1, 1*fc1_param*dimension22])

    h_fc1_re = tf.nn.relu(tf.add(tf.matmul(h_pool3_flat_re, variavl_dict["W_fc1"]), variavl_dict["b_fc1"]))
    y_conv_re=tf.add(tf.matmul(h_fc1_re,variavl_dict["W_fc4"]), variavl_dict["b_fc4"])
    #print y_conv_re.shape
    #cost =-tf.reshape(y_conv_re,[1])+tf.reduce_sum(tf.square(x_image_recon))/500.0
    #cost =-tf.reshape(tf.nn.sigmoid(y_conv_re[0][1])-tf.nn.sigmoid(y_conv_re[0][0]),[1])+tf.reduce_sum(tf.square(x_image_recon))/2000.0
    #cost =-tf.reshape(tf.nn.sigmoid(y_conv_re[0][1])-tf.nn.sigmoid(y_conv_re[0][0]),[1])*tf.log(tf.nn.sigmoid(y_conv_re[0][1]+0.000001)/(tf.nn.sigmoid(y_conv_re[0][0])+0.000001))+tf.reduce_sum(tf.square(x_image_recon))/2000.0
    #cost =-tf.reshape(tf.nn.sigmoid(y_conv_re[0][1])*tf.nn.sigmoid(y_conv_re[0][0]),[1])+tf.reduce_sum(tf.square(x_image_recon))/2000.0
    cost =-tf.reshape(tf.nn.sigmoid(y_conv_re[0][1])/(tf.nn.sigmoid(y_conv_re[0][0])+0.00001),[1])+tf.reduce_sum(tf.square(x_image_recon))/2000.0
    #cost =-y_conv_re+tf.reduce_sum(tf.square(x_image_recon))/500.0
    #cost =-(y_conv_re[0][1]-y_conv_re[0][0])+tf.reduce_sum(tf.square(x_image_recon))/2000.0

    #cost =-tf.reshape(y_conv_re,[1])
    train_step2 = tf.train.AdamOptimizer(0.001).minimize(cost)

    sess2.run(tf.global_variables_initializer())
    #x_image_recon.assign(positive_image[1])
    cost_list=[]
    for i in range(100000):
        _, progress, y_val, a, b  =sess2.run([train_step2, cost, y_conv_re, h_conv11_re_, h_conv22_re])
        cost_list.append(progress)
        if (i+1)%100==0:
            print('step '+str(i)+' cost: '+str(progress)+', y_conv_re: '+str(y_val))
        #if progress<=-0.9999:
            #break
        if i>20000:
            if cost_list[i-500]-cost_list[i]<=0.0000001:
                print(cost_list[i-100]-cost_list[i])
                break
        #print str(a.shape)+'\n'+str(b.shape)+'\n'+str(1*fc1_param*dimension3)

            #print a
    final_recon=x_image_recon.eval(session=sess2)
    final_recon_res=np.reshape(final_recon, (data_length, 4))
    inputdir=input_dir.split('/')[-1]
    np.savez_compressed(str(trained_model)+str(start_at), recon=final_recon_res)

    fig = plt.figure(figsize=(8,8))
        # Plot distance matrix.
    axmatrix = fig.add_axes([0.1,0.05,0.6,0.9])
    im = axmatrix.matshow(final_recon_res, aspect='auto', origin='lower', cmap=plt.get_cmap('YlGnBu'))
    axmatrix.set_xticks([])
    axmatrix.set_yticks([])
    axcolor = fig.add_axes([0.8,0.05,0.02,0.9])
    pylab.colorbar(im, cax=axcolor)
    fig.savefig(str(trained_model)+'_max_act_'+str(start_at)+'.png')
    import deepgmap.post_train_tools.sequence_visualizer2 as sq
    sq.seuquence_visualizer2(final_recon_res, str(trained_model)+'_max_act_seq_'+str(start_at)+'.pdf')
    plt.show()
    sess2.close()
else:
    print("don't understand the "+str(deconv)+" option")

max_stars_repo_name: koonimaru/DeepGMAP

max_issues_repo_name: koonimaru/DeepGMAP

max_forks_repo_name: koonimaru/DeepGMAP


text:
# TODO: Add import statements
import pandas as pd
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import PolynomialFeatures
import numpy as np
# Assign the data to predictor and outcome variables
# TODO: Load the data
train_data = pd.read_csv("data.csv")
X = train_data['Var_X'].values.reshape(-1,1)
y = train_data['Var_Y'].values

# Create polynomial features
# TODO: Create a PolynomialFeatures object, then fit and transform the
# predictor feature
poly_feat = PolynomialFeatures(degree=4)
X_poly = poly_feat.fit_transform(X)

# Make and fit the polynomial regression model
# TODO: Create a LinearRegression object and fit it to the polynomial predictor
# features
poly_model = LinearRegression(fit_intercept=False).fit(X_poly, y)

# Once you've completed all of the steps, select Test Run to see your model
# predictions against the data, or select Submit Answer to check if the degree
# of the polynomial features is the same as ours!

max_stars_repo_path: Supervised Learning/d) Polynomial Regression/Solution poly_reg.py


text:
import numpy as np

from compimg.windows import IdentitySlidingWindow

class TestIdentitySlidingWindow:
    def test_slide_correctness(self):
        input_image = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
        slider = IdentitySlidingWindow((2, 2), (1, 1))
        slides = list(slider.slide(input_image))
        assert len(slides) == 4
        assert np.array_equal(slides[0], np.array([[1, 2], [4, 5]]))
        assert np.array_equal(slides[1], np.array([[2, 3], [5, 6]]))
        assert np.array_equal(slides[2], np.array([[4, 5], [7, 8]]))
        assert np.array_equal(slides[3], np.array([[5, 6], [8, 9]]))

max_stars_repo_name: CaramelCake/compimg

max_issues_repo_name: CaramelCake/compimg

max_forks_repo_name: khrynczenko/compimg


text:
import jax.numpy as np
from jax import random
from jax import nn

def select_feats(graph_mat, num_nodes: int):
    return graph_mat[:, num_nodes:]

def select_adj(graph_mat, num_nodes: int):
    return graph_mat[:, :num_nodes]

def RnaGraphEmbedding(num_nodes: int, embedding_size: int):
    vocab_size = 4

    def init_fun(rng, input_shape):
        """
        :param input_shape: (num_nodes, num_nodes + num_features)
        """
        num_nodes, num_nodes_features = input_shape
        num_features = num_nodes_features - num_nodes

        embedding_matrix = random.normal(rng, shape=(vocab_size, embedding_size))
        # Add a zeros vector to the beginning for padded vector.
        embedding_matrix = np.concatenate(
            [np.zeros((1, embedding_size)), embedding_matrix]
        )
        return (
            num_nodes,
            num_nodes + embedding_size,
        ), embedding_matrix

    def apply_fun(params, inputs, **kwargs):
        """
        :param inputs: The node feature matrix.
            We assume that the node feature matrix's first column
            is the embedding index.
        """
        embedding_matrix = params
        adj = select_adj(inputs, num_nodes)
        feats = select_feats(inputs, num_nodes)

        indices = np.take(feats, 0, axis=1).astype(int)
        embedding = np.take(embedding_matrix, indices, axis=0)

        output = np.concatenate([adj, embedding], axis=1)
        return output

    return init_fun, apply_fun

from jax import lax

def NodeFeatureExtractor(num_nodes: int):
    def init_fun(rng, input_shape):
        num_nodes, num_feats = input_shape
        return (
            num_nodes,
            num_nodes + input_shape[-1] - 1,
        ), ()

    def apply_fun(params, inputs, **kwargs):
        adj = select_adj(inputs, num_nodes)
        feats = select_feats(inputs, num_nodes)

        return np.concatenate([adj, feats[:, 1:]], axis=1)

    return init_fun, apply_fun

from jax.tree_util import tree_map
from functools import partial
from drosha_gnn.layers import select_adj, select_feats

def GraphFanInConcat(num_nodes: int, axis: int = -1):
    def init_fun(rng, input_shape):

        num_feats = np.sum(np.array([i[1] - num_nodes for i in input_shape]))
        return (num_nodes, num_nodes + num_feats), ()

    def apply_fun(params, inputs, **kwargs):
        adj = tree_map(partial(select_adj, num_nodes=170), inputs)
        feats = tree_map(partial(select_feats, num_nodes=170), inputs)
        feats = np.concatenate(feats, axis=1)
        return np.concatenate([adj[0], feats], axis=1)

    return init_fun, apply_fun

from functools import partial
from jax import vmap

def concat_nodes(node1, node2):
    """Concatenate two nodes together."""
    return np.concatenate([node1, node2])

def concatenate(node: np.ndarray, node_feats: np.ndarray):
    """Concatenate node with each node in node_feats.

    Behaviour is as follows.
    Given a node with features `f_0` and stacked node features
    `[f_0, f_1, f_2, ..., f_N]`,
    return a stacked concatenated feature array:
    `[(f_0, f_0), (f_0, f_1), (f_0, f_2), ..., (f_0, f_N)]`.

    :param node: A vector embedding of a single node in the graph.
        Should be of shape (n_input_features,)
    :param node_feats: Stacked vector embedding of all nodes in the graph.
        Should be of shape (n_nodes, n_input_features)
    :returns: A stacked array of concatenated node features.
    """
    return vmap(partial(concat_nodes, node))(node_feats)

def concatenate_node_features(node_feats):
    """Return node-by-node concatenated features.

    Given a node feature matrix of shape (n_nodes, n_features),
    this returns a matrix of shape (n_nodes, n_nodes, 2*n_features).
    """
    outputs = vmap(partial(concatenate, node_feats=node_feats))(node_feats)
    return outputs

def attentive_adjacency(params, adj, feats, num_nodes):
    w1, b1, w2, b2, wa = params
    node_by_node_concat = concatenate_node_features(feats)

    # Neural network piece here.
    a1 = np.tanh(np.dot(node_by_node_concat, w1) + b1)
    # a2 = np.dot(a1, w2) + b2
    a2 = nn.elu(np.dot(a1, w2) + b2)
    return a2 * adj

    # Finally, we multiply by a matrix of the same shape
    # to get an attention matrix that doesn't have banding.
    # attentive_adj = adj * a2 * wa
    # return attentive_adj

def AttentiveMessagePassingLayer(num_nodes: int, hidden_dims: int = 256):
    """Attentive message passing on a graph.

    We use a feed forward neural network to learn
    the weights on which a message passing operator should work.

    The input is the graph matrix. Should be of size (num_nodes, num_nodes + num_feats).
    The output is also of the size (num_nodes, num_nodes + num_feats).
    """

    def init_fun(rng, input_shape):
        num_nodes, n_node_feats = input_shape
        num_feats = n_node_feats - num_nodes
        k1, k2, k3, k4, k5 = random.split(rng, 5)

        # Params for neural network transformation of node concatenated features.
        w1 = random.normal(k1, shape=(2 * num_feats, hidden_dims)) * 0.001
        b1 = random.normal(k2, shape=(hidden_dims,)) * 0.001
        w2 = random.normal(k3, shape=(hidden_dims,)) * 0.001
        b2 = random.normal(k4, shape=(1,)) * 0.001
        wa = random.normal(k5, shape=(num_nodes, num_nodes))

        params = w1, b1, w2, b2, wa
        output_shape = (num_nodes, num_nodes + num_feats)
        return output_shape, params

    def apply_fun(params, inputs, **kwargs):
        ### START ATTENTIVE MATRIX CALCULATION ###
        adj = select_adj(inputs, num_nodes)
        feats = select_feats(inputs, num_nodes)
        attentive_adj = attentive_adjacency(params, adj, feats, num_nodes)
        ### END ATTENTIVE MATRIX CALCULATION ###
        mp = np.dot(attentive_adj, feats)
        return np.concatenate([adj, mp], axis=1)

    return init_fun, apply_fun

def GetGraphAdjAttentionMatrix(num_nodes: int, hidden_dims: int = 256):
    """Return just the attentive matrix.

    Intended to be the final layer of a model.

    We use a feed forward neural network to learn
    the weights on which a message passing operator should work.

    The input is the graph matrix. Should be of size (num_nodes, num_nodes + num_feats).
    The output is also of the size (num_nodes, num_nodes + num_feats).
    """

    def init_fun(rng, input_shape):
        num_nodes, n_node_feats = input_shape
        num_feats = n_node_feats - num_nodes
        k1, k2, k3, k4, k5 = random.split(rng, 5)

        # Params for neural network transformation of node concatenated features.
        w1 = random.normal(k1, shape=(2 * num_feats, hidden_dims)) * 0.001
        b1 = random.normal(k2, shape=(hidden_dims,)) * 0.001
        w2 = random.normal(k3, shape=(hidden_dims,)) * 0.001
        b2 = random.normal(k4, shape=(1,)) * 0.001
        wa = random.normal(k5, shape=(num_nodes, num_nodes))

        params = w1, b1, w2, b2, wa
        output_shape = (num_nodes, num_nodes + num_feats)
        return output_shape, params

    def apply_fun(params, inputs, **kwargs):
        adj = select_adj(inputs, num_nodes)
        feats = select_feats(inputs, num_nodes)
        attentive_adj = attentive_adjacency(params, adj, feats, num_nodes)
        return attentive_adj

    return init_fun, apply_fun

def compute_node_attention_weights(params, feats):
    w1, b1, w2, b2 = params
    a1 = np.tanh(np.dot(feats, w1) + b1)
    a2 = nn.elu(np.dot(a1, w2) + b2)
    return np.squeeze(a2)

def AttentiveGraphSummation(num_nodes, hidden_dims: int = 2048):
    def init_fun(rng, input_shape):
        num_nodes, num_node_feats = input_shape
        num_feats = num_node_feats - num_nodes

        k1, k2, k3, k4 = random.split(rng, 4)
        # Params for neural network transformation of node concatenated features.
        w1 = random.normal(k1, shape=(num_feats, hidden_dims)) * 0.001
        b1 = random.normal(k2, shape=(hidden_dims,)) * 0.001
        w2 = random.normal(k3, shape=(hidden_dims, 1)) * 0.001
        b2 = random.normal(k4, shape=(1,)) * 0.001
        params = w1, b1, w2, b2
        output_shape = (num_feats,)
        return output_shape, params

    def apply_fun(params, inputs, **kwargs):
        feats = select_feats(inputs, num_nodes)

        # Neural network piece here.
        node_attn_weights = compute_node_attention_weights(params, feats)

        # Weighted summation happens here
        out = np.dot(node_attn_weights, feats)
        return out

    return init_fun, apply_fun

def GetNodeAttention(num_nodes, hidden_dims: int = 16):
    def init_fun(rng, input_shape):
        num_nodes, num_node_feats = input_shape
        num_feats = num_node_feats - num_nodes

        k1, k2, k3, k4 = random.split(rng, 4)
        # Params for neural network transformation of node concatenated features.
        w1 = random.normal(k1, shape=(num_feats, hidden_dims)) * 0.001
        b1 = random.normal(k2, shape=(hidden_dims,)) * 0.001
        w2 = random.normal(k3, shape=(hidden_dims, 1)) * 0.001
        b2 = random.normal(k4, shape=(1,)) * 0.001
        params = w1, b1, w2, b2
        output_shape = (num_feats,)
        return output_shape, params

    def apply_fun(params, inputs, **kwargs):
        feats = select_feats(inputs, num_nodes)

        # Neural network piece here.
        node_attn_weights = compute_node_attention_weights(params, feats)
        return node_attn_weights

    return init_fun, apply_fun

max_stars_repo_name: ericmjl/drosha-gnn

max_issues_repo_name: ericmjl/drosha-gnn

max_forks_repo_name: ericmjl/drosha-gnn

reason: import jax,from jax


text:
import random
import os
import re
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer
from sklearn import linear_model
from sklearn import tree
from sklearn import svm
# PUT POLARITY DATASET PATH HERE
POLARITY_PATH = '/Users/marcotcr/phd/datasets/multi_domain_polarity/'
def LoadDataset(dataset_name):
  if dataset_name.endswith('ng'):
    if dataset_name == '2ng':
      cats = ['alt.atheism', 'soc.religion.christian']
      class_names = ['Atheism', 'Christianity']
    if dataset_name == 'talkng':
      cats = ['talk.politics.guns', 'talk.politics.misc']
      class_names = ['Guns', 'PoliticalMisc']
    if dataset_name == '3ng':
      cats = ['comp.os.ms-windows.misc', 'comp.sys.ibm.pc.hardware', 'comp.windows.x']
      class_names = ['windows.misc', 'ibm.hardware', 'windows.x']
    newsgroups_train = fetch_20newsgroups(subset='train',categories=cats)
    newsgroups_test = fetch_20newsgroups(subset='test',categories=cats)
    train_data = newsgroups_train.data
    train_labels = newsgroups_train.target
    test_data = newsgroups_test.data
    test_labels = newsgroups_test.target
    return train_data, train_labels, test_data, test_labels, class_names
  if dataset_name.startswith('multi_polarity_'):
    name = dataset_name.split('_')[2]
    return LoadMultiDomainDataset(POLARITY_PATH + name)
def LoadMultiDomainDataset(path_data, remove_bigrams=True):
  random.seed(1)
  pos = []
  neg = []
  def get_words(line, remove_bigrams=True):
    z = [tuple(x.split(':')) for x in re.findall('\w*?:\d', line)]
    if remove_bigrams:
      z = ' '.join([' '.join([x[0]] * int(x[1])) for x in z if '_' not in x[0]])
    else:
      z = ' '.join([' '.join([x[0]] * int(x[1])) for x in z])
    return z
  for line in open(os.path.join(path_data, 'negative.review')):
    neg.append(get_words(line, remove_bigrams))
  for line in open(os.path.join(path_data, 'positive.review')):
    pos.append(get_words(line, remove_bigrams))
  random.shuffle(pos)
  random.shuffle(neg)
  split_pos = int(len(pos) * .8)
  split_neg = int(len(neg) * .8)
  train_data = pos[:split_pos] + neg[:split_neg]
  test_data = pos[split_pos:] + neg[split_neg:]
  train_labels = [1] * len(pos[:split_pos]) + [0] * len(neg[:split_neg])
  test_labels = [1] * len(pos[split_pos:]) + [0] * len(neg[split_neg:])
  return train_data, np.array(train_labels), test_data, np.array(test_labels), ['neg', 'pos']

max_stars_repo_name: marcotcr/lime-kdd

max_issues_repo_name: marcotcr/lime-kdd

max_forks_repo_name: marcotcr/lime-kdd


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from __future__ import print_function
import os
import numpy as np
import SimpleITK as sitk
import scipy.misc
from skimage.transform import resize
# from scipy.misc import imresize
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
from scipy import ndimage

import cv2
import time
from decimal import Decimal
import skimage.io as io

data_dir = '/Users/chenjingkun/Documents/data/C0LET2_nii45_for_challenge19/c0t2lge/'
thresh = 1
new_shape = (480,480)
rows = 256
cols = 256
xmin = 1
xmax = 1
ymin = 1
ymax = 1
xlenmin = 1
ylenmin = 1
img_count = 0
def show_img(data):
    # for i in range(data.shape[0]):
    #     io.imshow(data[i, :, :], cmap='gray')
        io.imshow(data[:,:], cmap = 'gray')
        io.show()
def show_img_all(data):
    for i in range(data.shape[0]):
        io.imshow(data[i, :, :], cmap='gray')
        # io.imshow(data[:,:], cmap = 'gray')
        io.show()

# label transform, 500-->1, 200-->2, 600-->3

###### LGE
LGE_data_1ch = []
LGE_gt_1ch = []
img_dir = '/Users/chenjingkun/Documents/data/C0LET2_nii45_for_challenge19/lge_images/'
if not os.path.exists(img_dir):
    os.makedirs(img_dir)
gt_dir_1 = '/Users/chenjingkun/Documents/data/C0LET2_nii45_for_challenge19/lgegt/'
lge_list = []

for pp in range(1, 6):

    data_name = data_dir + 'patient' + str(pp) + '_LGE.nii.gz'
    gt_name = gt_dir_1 + 'patient' + str(pp) + '_LGE_manual.nii.gz'
    img = sitk.ReadImage(os.path.join(gt_name))
    data_array = sitk.GetArrayFromImage(sitk.ReadImage(
        os.path.join(data_name)))
    gt_array = sitk.GetArrayFromImage(sitk.ReadImage(os.path.join(gt_name)))
    img_count +=gt_array.shape[0]

    print(np.shape(data_array))

    x = []
    y = []
    new_gt_list = []
    for image in gt_array:
        image = np.asarray(image)
        image1 = image.copy()
        image2 = image.copy()
        image[image == 500] = 1
        image[image == 200] = 0
        image[image == 600] = 0
        image1[image1 == 500] = 0
        image1[image1 == 200] = 1
        image1[image1 == 600] = 0
        image2[image2 == 500] = 0
        image2[image2 == 200] = 0
        image2[image2 == 600] = 1

        image = resize(image,new_shape, preserve_range =True)
        image1 = resize(image1,new_shape, preserve_range =True)
        image2 = resize(image2,new_shape, preserve_range =True)
        image = np.around(image)
        image1 = np.around(image1)
        image2 = np.around(image2)
        image = image.astype(np.int32)
        image1 = image1.astype(np.int32)
        image2 = image2.astype(np.int32)

        image[image == 1] = 1
        image1[image1 == 1] = 2
        image2[image2 == 1] = 3
        image = image +image1 +image2
        [x_test, y_test] = image.shape
        for i in range(x_test):
            for j in range(y_test):
                if(image[i, j] >3) :
                    print("--------error----------:", pp)
        image[image == 1] = 500
        image[image == 2] = 200
        image[image == 3] = 600
        for i in range(image.shape[0]):
            for j in range(image.shape[1]):
                if image[i][j] != 0:
                    if j < 40 or i < 40:
                        image[0:200, 0:50] = 0
                    else:
                        x.append(i)
                        y.append(j)
        new_gt_list.append(image)

    gt_array=np.array(new_gt_list)
    print("new_array:",gt_array.shape)

    new_data_list = []
    print("idx:", pp)
    for image in data_array:
        image = np.asarray(image)
        image = resize(image, new_shape, preserve_range =True)
        image = np.around(image)
        image = image.astype(np.int32)
        new_data_list.append(image)

    data_array=np.array(new_data_list)

    print(min(x),max(x),max(x)-min(x),round(min(x)/np.shape(gt_array)[1],2), round(max(x)/np.shape(gt_array)[1],2))
    print(min(y),max(y),max(y)-min(y),round(min(y)/np.shape(gt_array)[1],2), round(max(y)/np.shape(gt_array)[1],2))

    mask = np.zeros(np.shape(data_array), dtype='float32')
    mask[data_array >= thresh] = 1
    mask[data_array < thresh] = 0
    for iii in range(np.shape(data_array)[0]):
        mask[iii, :, :] = scipy.ndimage.morphology.binary_fill_holes(
            mask[iii, :, :])  #fill the holes inside br
    data_array = data_array - np.mean(data_array[mask == 1])
    data_array /= np.std(data_array[mask == 1])
    rows_o = np.shape(data_array)[1]
    cols_o = np.shape(data_array)[2]

    data_array_ = data_array[:,
                             int((rows_o - rows) /
                                 2):int((rows_o - rows) / 2) + rows,
                             int((cols_o - cols) /
                                 2):int((cols_o - cols) / 2) + cols]
    gt_array_ = gt_array[:,
                         int((rows_o - rows) /
                             2):int((rows_o - rows) / 2) + rows,
                         int((cols_o - cols) / 2):int((cols_o - cols) / 2) +
                         cols]

    LGE_data_1ch.extend(np.float32(data_array_))
    LGE_gt_1ch.extend(np.float32(gt_array_))

LGE_data_1ch = np.asarray(LGE_data_1ch)
LGE_gt_1ch = np.asarray(LGE_gt_1ch)
LGE_gt_1ch[LGE_gt_1ch == 500] = 1
LGE_gt_1ch[LGE_gt_1ch == 200] = 2
LGE_gt_1ch[LGE_gt_1ch == 600] = 3

# np.save('LGE_data_1ch_old.npy', LGE_data_1ch)
# np.save('LGE_gt_1ch_old.npy', LGE_gt_1ch)

##### T2
T2_data_1ch = []
T2_gt_1ch = []
T2_shape = []
img_dir = '/Users/chenjingkun/Documents/data/C0LET2_nii45_for_challenge19/t2_images/'
if not os.path.exists(img_dir):
    os.makedirs(img_dir)
gt_dir_1 = '/Users/chenjingkun/Documents/data/C0LET2_nii45_for_challenge19/t2gt/'

for pp in range(1, 36):
    data_name = data_dir + 'patient' + str(pp) + '_T2.nii.gz'
    gt_name = gt_dir_1 + 'patient' + str(pp) + '_T2_manual.nii.gz'

    data_array = sitk.GetArrayFromImage(sitk.ReadImage(
        os.path.join(data_name)))
    gt_array = sitk.GetArrayFromImage(sitk.ReadImage(os.path.join(gt_name)))

    data_array = np.nan_to_num(data_array, copy=True)
    gt_array = np.nan_to_num(gt_array, copy=True)

    img_count +=gt_array.shape[0]
    x = []
    y = []
    count = 0
    print("idx:", pp)
    new_gt_list = []
    for image in gt_array:
        image = np.asarray(image)
        image1 = image.copy()
        image2 = image.copy()
        image[image == 500] = 1
        image[image == 200] = 0
        image[image == 600] = 0
        image1[image1 == 500] = 0
        image1[image1 == 200] = 1
        image1[image1 == 600] = 0
        image2[image2 == 500] = 0
        image2[image2 == 200] = 0
        image2[image2 == 600] = 1

        image = resize(image,new_shape, preserve_range =True)
        image1 = resize(image1,new_shape, preserve_range =True)
        image2 = resize(image2,new_shape, preserve_range =True)

        image = np.around(image)
        image1 = np.around(image1)
        image2 = np.around(image2)
        image = image.astype(np.int32)
        image1 = image1.astype(np.int32)
        image2 = image2.astype(np.int32)

        image[image == 1] = 1
        image1[image1 == 1] = 2
        image2[image2 == 1] = 3
        image = image +image1 +image2
        [x_test, y_test] = image.shape
        for i in range(x_test):
            for j in range(y_test):
                if(image[i, j] >3) :
                    print("--------error----------:", pp)
        image[image == 1] = 500
        image[image == 2] = 200
        image[image == 3] = 600

        for i in range(image.shape[0]):
            for j in range(image.shape[1]):
                if image[i][j] != 0:
                    if j < 40 or i < 40:
                        image[0:200, 0:50] = 0
                    else:
                        x.append(i)
                        y.append(j)
        new_gt_list.append(image)
    print("new_gt_list:",len(new_gt_list))

    gt_array=np.array(new_gt_list)
    print("new_array:",gt_array.shape)

    new_data_list = []
    for image in data_array:
        image = np.asarray(image)
        image = resize(image, new_shape, preserve_range =True)
        image = np.around(image)
        image = image.astype(np.int32)
        new_data_list.append(image)
    data_array=np.array(new_data_list)
    print(min(x), max(x),
          max(x) - min(x), round(min(x) / np.shape(gt_array)[1], 2),
          round(max(x) / np.shape(gt_array)[1], 2))
    print(min(y), max(y),
          max(y) - min(y), round(min(y) / np.shape(gt_array)[1], 2),
          round(max(y) / np.shape(gt_array)[1], 2))
    if(round(min(x)/np.shape(gt_array)[1],2) < 0.2 or round(min(y)/np.shape(gt_array)[1],2)<0.2):
        print("errorerrorerrorerrorerrorerror")
        show_img(gt_array)
    mask = np.zeros(np.shape(data_array), dtype='float32')
    mask[data_array >= thresh] = 1
    mask[data_array < thresh] = 0

    for iii in range(np.shape(data_array)[0]):
        mask[iii, :, :] = scipy.ndimage.morphology.binary_fill_holes(
            mask[iii, :, :])  #fill the holes inside br

    data_array = data_array - np.mean(data_array[mask == 1])
    data_array /= np.std(data_array[mask == 1])
    rows_o = np.shape(data_array)[1]
    cols_o = np.shape(data_array)[2]

    data_array_ = data_array[:,
                             int((rows_o - rows) /
                                 2):int((rows_o - rows) / 2) + rows,
                             int((cols_o - cols) /
                                 2):int((cols_o - cols) / 2) + cols]
    gt_array_ = gt_array[:,
                         int((rows_o - rows) /
                             2):int((rows_o - rows) / 2) + rows,
                         int((cols_o - cols) / 2):int((cols_o - cols) / 2) +
                         cols]

    T2_data_1ch.extend(np.float32(data_array_))
    T2_gt_1ch.extend(np.float32(gt_array_))

T2_data_1ch = np.asarray(T2_data_1ch)
T2_gt_1ch = np.asarray(T2_gt_1ch)
T2_gt_1ch[T2_gt_1ch == 500] = 1
T2_gt_1ch[T2_gt_1ch == 200] = 2
T2_gt_1ch[T2_gt_1ch == 600] = 3

#######C0
#
C0_data_1ch = []
C0_gt_1ch = []
img_dir = '/Users/chenjingkun/Documents/data/C0LET2_nii45_for_challenge19/c0_images/'
if not os.path.exists(img_dir):
    os.makedirs(img_dir)
gt_dir_1 = '/Users/chenjingkun/Documents/data/C0LET2_nii45_for_challenge19/c0gt/'

for pp in range(1, 36):
    data_name = data_dir + 'patient' + str(pp) + '_C0.nii.gz'
    gt_name = gt_dir_1 + 'patient' + str(pp) + '_C0_manual.nii.gz'
    data_array = sitk.GetArrayFromImage(sitk.ReadImage(
        os.path.join(data_name)))
    gt_array = sitk.GetArrayFromImage(sitk.ReadImage(os.path.join(gt_name)))
    print(np.shape(data_array))
    img_count +=gt_array.shape[0]

    x = []
    y = []
    for image in gt_array:
        for i in range(image.shape[0]):
            for j in range(image.shape[1]):
                if image[i][j] != 0:
                    if i < 30 or j <30:
                        print("label_error:", pp,image.shape)
                    else:
                        x.append(i)
                        y.append(j)
    new_gt_list = []
    for image in gt_array:
        image = np.asarray(image)
        image1 = image.copy()
        image2 = image.copy()
        image[image == 500] = 1
        image[image == 200] = 0
        image[image == 600] = 0
        image1[image1 == 500] = 0
        image1[image1 == 200] = 1
        image1[image1 == 600] = 0
        image2[image2 == 500] = 0
        image2[image2 == 200] = 0
        image2[image2 == 600] = 1

        image = resize(image,new_shape, preserve_range =True)
        image1 = resize(image1,new_shape, preserve_range =True)
        image2 = resize(image2,new_shape, preserve_range =True)

        image = np.around(image)
        image1 = np.around(image1)
        image2 = np.around(image2)
        image = image.astype(np.int32)
        image1 = image1.astype(np.int32)
        image2 = image2.astype(np.int32)

        image[image == 1] = 1
        image1[image1 == 1] = 2
        image2[image2 == 1] = 3
        image = image +image1 +image2
        [x_test, y_test] = image.shape
        for i in range(x_test):
            for j in range(y_test):
                if(image[i, j] >3) :
                    print("--------error----------:", pp)
        image[image == 1] = 500
        image[image == 2] = 200
        image[image == 3] = 600

        for i in range(image.shape[0]):
            for j in range(image.shape[1]):
                if image[i][j] != 0:
                    if j < 40 or i < 40:
                        image[0:200, 0:50] = 0
                    else:
                        x.append(i)
                        y.append(j)
        new_gt_list.append(image)
    print("new_gt_list:",len(new_gt_list))

    gt_array=np.array(new_gt_list)
    print("new_array:",gt_array.shape)

    new_data_list = []
    for image in data_array:
        image = np.asarray(image)
        image = resize(image, new_shape, preserve_range =True)
        image = np.around(image)
        image = image.astype(np.int32)
        new_data_list.append(image)
    data_array=np.array(new_data_list)

    print("idx:", pp)
    print(min(x), max(x),
          max(x) - min(x), round(min(x) / np.shape(gt_array)[1], 2),
          round(max(x) / np.shape(gt_array)[1], 2))
    print(min(y), max(y),
          max(y) - min(y), round(min(y) / np.shape(gt_array)[1], 2),
          round(max(y) / np.shape(gt_array)[1], 2))

    mask = np.zeros(np.shape(data_array), dtype='float32')
    mask[data_array >= thresh] = 1
    mask[data_array < thresh] = 0

    for iii in range(np.shape(data_array)[0]):
        mask[iii, :, :] = scipy.ndimage.morphology.binary_fill_holes(
            mask[iii, :, :])  #fill the holes inside br
    data_array = data_array - np.mean(data_array[mask == 1])
    data_array /= np.std(data_array[mask == 1])
    rows_o = np.shape(data_array)[1]
    cols_o = np.shape(data_array)[2]

    data_array_ = data_array[:,
                             int((rows_o - rows) /
                                 2):int((rows_o - rows) / 2) + rows,
                             int((cols_o - cols) /
                                 2):int((cols_o - cols) / 2) + cols]
    gt_array_ = gt_array[:,
                         int((rows_o - rows) /
                             2):int((rows_o - rows) / 2) + rows,
                         int((cols_o - cols) / 2):int((cols_o - cols) / 2) +
                         cols]

    C0_data_1ch.extend(np.float32(data_array_))
    C0_gt_1ch.extend(np.float32(gt_array_))

C0_data_1ch = np.asarray(C0_data_1ch)
C0_gt_1ch = np.asarray(C0_gt_1ch)
C0_gt_1ch[C0_gt_1ch == 500] = 1
C0_gt_1ch[C0_gt_1ch == 200] = 2
C0_gt_1ch[C0_gt_1ch == 600] = 3
print("LGE_data_1ch:", LGE_data_1ch.shape)
print("C0_data_1ch:", C0_data_1ch.shape)
print("T2_data_1ch:", T2_data_1ch.shape)
print("LGE_gt_1ch:", LGE_gt_1ch.shape)
print("C0_gt_1ch:", C0_gt_1ch.shape)
print("T2_gt_1ch:", T2_gt_1ch.shape)

new_data_array = np.concatenate((LGE_data_1ch, C0_data_1ch), axis=0)
new_data_array = np.concatenate((new_data_array, LGE_data_1ch), axis=0)
new_data_array = np.concatenate((new_data_array, T2_data_1ch), axis=0)
new_gt_array = np.concatenate((LGE_gt_1ch, C0_gt_1ch), axis=0)
new_gt_array = np.concatenate((new_gt_array, LGE_gt_1ch), axis=0)
new_gt_array = np.concatenate((new_gt_array, T2_gt_1ch), axis=0)
print("new_gt_array:", new_gt_array.shape)
print("new_gt_array:", new_gt_array.shape)
train_imgs_new  = new_data_array.copy()
train_masks_new  = new_gt_array.copy()
count_i = 0
count = 0
count_list = []
for image in new_gt_array:
    max_1 = np.max(image)

    if max_1 < 0:
        delete_number = count_i - count
        train_imgs_new = np.delete(train_imgs_new, delete_number, axis=0)
        train_masks_new = np.delete(train_masks_new, delete_number, axis=0)

        count += 1
        print("empty:",count, count_i)

    count_i +=1
new_data_array = train_imgs_new
new_gt_array = train_masks_new
print("new_gt_array:", new_gt_array.shape)
print("new_gt_array:", new_gt_array.shape)

# np.save('/Users/chenjingkun/Documents/result/MS-CMR_miccai_2019_result/del/all_data_resize_256_256.npy', new_data_array[:, :, :, np.newaxis])
# np.save('/Users/chenjingkun/Documents/result/MS-CMR_miccai_2019_result/del/all_gt_resize_256_256.npy', new_gt_array[:, :, :, np.newaxis])
# output_path = "/Users/chenjingkun/Documents/result/MS-CMR_miccai_2019_result/del/all_data_resize_256_256.nii.gz"
# sitk.WriteImage(sitk.GetImageFromArray(new_data_array),output_path)
# output_path = "/Users/chenjingkun/Documents/result/MS-CMR_miccai_2019_result/del/all_gt_resize_256_256.nii.gz"
# sitk.WriteImage(sitk.GetImageFromArray(new_gt_array),output_path)
print("img_count:",img_count)
print("new_gt_array:",new_gt_array.shape)


text:
import constants
import discord
import re
from discord.ext import commands
from utils import discord_utils, logging_utils
from modules.music_race import music_race_constants
import os
import numpy as np
import string

def get_partition_mapping():
    map = {}
    for answer in music_race_constants.ANSWERS:
        for idx, letter in enumerate(answer):
            if letter in map:
                map[letter].append(f"{answer}_part_{idx}")
            else:
                map[letter] = [f"{answer}_part_{idx}"]
    for letter in filter(lambda x: x not in map, string.ascii_uppercase):
        map[letter] = ["silence"]
    return map

class MusicRace(commands.Cog, name="Music Race"):
    """Puzzle for Arithmancy June 2020! Identify each of the movies based on their theme songs!"""
    def __init__(self, bot):
        self.bot = bot
        self.partition_map = get_partition_mapping()

    @commands.command(name="hint")
    async def hint(self, ctx):
        """Hint

        Usage: `~hint`
        """
        logging_utils.log_command("hint", ctx.guild, ctx.channel, ctx.author)
        embed = discord_utils.create_embed()
        embed.add_field(name="This is not a hint",
                        value="*Hints will always be given at Hogwarts to those who ask for it.*")
        await ctx.send(embed=embed)

    @commands.command(name="notesaw", aliases=["musicpuzzleinfo"])
    async def musicpuzzleinfo(self, ctx):
        """Give the users everything they need to know about the puzzle

        Usage: `~notesaw`
        """
        logging_utils.log_command("notesaw", ctx.guild, ctx.channel, ctx.author)
        embed = discord_utils.create_embed()

        embed.add_field(name=f"Welcome to Notesaw!",
                        value=f"To start the puzzle, use `{ctx.prefix}guesstune`. "
                              f"For example, try `{ctx.prefix}guesstune PIANO`. Have fun!",
                        inline=False)
        embed.add_field(name=f"Notice",
                        value=f"Headphone users! We recommend turning the volume down a bit. Some minor glitches might "
                              f"hurt your ear",
                        inline=False)
        await ctx.send(embed=embed)

    @commands.command(name="guesstune")
    async def guesstune(self, ctx, *args):
        """Take a user's guess and give them a response based on what letters they provided

        Usage: `~guesstune (WORD)`
        """
        logging_utils.log_command("guesstune", ctx.guild, ctx.channel, ctx.author)
        embed = discord_utils.create_embed()

        if len(args) < 1:
            embed = discord_utils.create_no_argument_embed("word")
            await ctx.send(embed=embed)
            return

        # Replace any non-letters
        word = re.sub("[^A-Z]+", "", "".join(args).upper())

        if len(word) < 1 or len(word) > 20:
            embed.add_field(name=f"{constants.FAILED}!",
                            value=f"Word provided `{word}` is not between 1-20 letters")
            await ctx.send(embed=embed)
            return

        if word in music_race_constants.ANSWERS:
            final_song_path = os.path.join(music_race_constants.PUZZLE_OUTPUTS_DIR, word + f"_final{music_race_constants.MP3_EXTENSION}")
            if not os.path.exists(final_song_path):
                delay = music_race_constants.ANSWERS[word][music_race_constants.DELAY] * 1000
                os.system(
                    f"ffmpeg -y -hide_banner -loglevel error -i {os.path.join(music_race_constants.PUZZLE_FULL_SONGS_DIR, word + music_race_constants.MP3_EXTENSION)} -filter_complex 'adelay={delay}|{delay}' {final_song_path}"
                )
                # TODO: ffmpeg-normalize is too slow for now. Try to optimize later.
                # os.system(
                #    f"ffmpeg-normalize -f -c:a libmp3lame {output_path} -o {output_path}"
                # )
            embed.add_field(name=f"Well done! You guessed {word}!",
                            value=f"\nTo play this tune yourself, use this command. (See {ctx.prefix}playtunehelp for more help)"
                            f"\n\n`{music_race_constants.ANSWERS[word][music_race_constants.TUNE]}`")
            await ctx.send(embed=embed)
            await ctx.send(
                file=discord.File(final_song_path,
                                  filename=f"{list(music_race_constants.ANSWERS).index(word)+1} of {len(music_race_constants.ANSWERS)}.mp3"))
            return

        # We need to figure out the longest substring that is part of the answer
        # Add that duration of the song.
        # Afterwards, we need to find all the remaining letters
        # and add random partitions, or silence.
        finalanswer = []
        delay = 0
        # For each letter we match, we can add that to the start of our output song
        for i in range(len(word)):
            found_song = False
            x = word[0:(i+1)]
            # Matches one of the songs
            for answer in music_race_constants.ANSWERS:
                if answer.startswith(x):
                    finalanswer.append((f"{answer}_part_{i}", delay))
                    found_song = True
                    break
            # Newly added character is not the start to a song
            if not found_song:
                # Get a random clip with that letter
                if word[i] in self.partition_map:
                    finalanswer.append((np.random.choice(self.partition_map[word[i]]), delay))
            # Increments
            delay += 3

        #debug_output_msg = ""
        #for ans in finalanswer:
        #    debug_output_msg += f"{ans[1]}-{ans[1]+3}: {ans[0]}\n"
        # TODO: Remove once we are more certain about how this works. It ruins the puzzle, obviously
        #await ctx.send(debug_output_msg)
        #print(word)
        #print(debug_output_msg)

        inputs = ''.join([f"-i {os.path.join(music_race_constants.PUZZLE_PARTIAL_SONGS_DIR, finalanswer[idx][0] + '.mp3')} " for idx in range(len(finalanswer))])
        # Otherwise, we just chop each song into 3s bits, with 0.5s between them
        filter_complex = "".join([f"[{idx}]atrim=0:{music_race_constants.SONG_SNIPPET_LENGTH},adelay={finalanswer[idx][1]*1000+500*idx}|{finalanswer[idx][1]*1000+500*idx},volume={music_race_constants.VOLUME/2}[{letter}];"
                                  for idx, letter in zip(range(len(finalanswer)), string.ascii_lowercase)])
        mix = ''.join([f"[{letter}]" for _, letter in zip(finalanswer, list(string.ascii_lowercase))])

        output_dir = os.path.join(music_race_constants.PUZZLE_OUTPUTS_DIR, ctx.channel.name)
        if not os.path.exists(output_dir):
            os.mkdir(output_dir)

        output_path = os.path.join(output_dir, f"{word}.mp3")
        os.system(
            f"ffmpeg -y -hide_banner -loglevel error {inputs} -preset veryfast " +
            f"-filter_complex '{filter_complex}{mix}amix=inputs={len(finalanswer)}:dropout_transition=1000,volume={music_race_constants.VOLUME/2},loudnorm' "
            f"{output_path}"
        )
        # TODO: ffmpeg-normalize is too slow for now. Try to optimize later.
        #os.system(
        #    f"ffmpeg-normalize -f -c:a libmp3lame {output_path} -o {output_path}"
        #)
        await ctx.send(file=discord.File(output_path))

def setup(bot):
    bot.add_cog(MusicRace(bot))

max_stars_repo_name: kevslinger/DiscordCipherRace

max_issues_repo_name: kevslinger/DiscordCipherRace

max_forks_repo_name: kevslinger/DiscordCipherRace