text:
# -*- coding: utf-8 -*-
"""
Created on Mon Jan 27 15:40:52 2020

@author: MichaelEK
"""
import os
import numpy as np
import pandas as pd
from pdsf.util import create_snowflake_engine, get_pk_stmt, compare_dfs, get_un_stmt
import snowflake.connector

####################################################

def read_table(username, password, account, database, schema, stmt, warehouse=None):
    """
    Function to import data from an MSSQL database.

    Parameters
    ----------
    username : str
        The username
    password : str
        The password
    account : str
        account is the name assigned to your account by Snowflake. In the hostname you received from Snowflake (after your account was provisioned), your account name is the full/entire string to the left of snowflakecomputing.com
    database : str
        The specific database within the server. e.g.: 'LowFlows'
    schema : str
        The schema associated with the table to be read.
    stmt : str
        Custom SQL statement to be directly passed to the database and schema.
    warehouse: str
        Optionally, set the snowflake warehouse to operate under

    Returns
    -------
    DataFrame
    """

    ## Create connection to database and execute sql statement
    ctx = snowflake.connector.connect(
        user=username,
        password=password,
        account=account,
        database=database,
        schema=schema,
        warehouse=warehouse
        )
    cs = ctx.cursor()
    try:
        cs.execute(stmt)
        df1 = cs.fetch_pandas_all()
    finally:
        cs.close()
        ctx.close()

    return df1

def to_table(df, table, username, password, account, database, schema, truncate=False, warehouse=None):
    """
    Function to append a DataFrame onto an existing mssql table.

    Parameters
    ----------
    df : DataFrame
        DataFrame to be saved. The DataFrame column/index names must match those on the mssql table exactly.
    table : str
        The specific table within the database. e.g.: 'LowFlowSiteRestrictionDaily'
    username : str
        The username
    password : str
        The password
    account : str
        account is the name assigned to your account by Snowflake. In the hostname you received from Snowflake (after your account was provisioned), your account name is the full/entire string to the left of snowflakecomputing.com
    database : str
        The specific database within the server. e.g.: 'LowFlows'
    schema : str
        The schema associated with the table to be read.
    truncate : bool
        Should the table rows be cleared via truncate before adding new data?
    warehouse: str
        Optionally, set the snowflake warehouse to operate under

    Returns
    -------
    None
    """
    if truncate:
        ctx = snowflake.connector.connect(
                user=username,
                password=password,
                account=account,
                database=database,
                schema=schema,
                warehouse=warehouse
                )
        cs = ctx.cursor()
        try:
            cs.execute('truncate "{schema}"."{table}"'.format(schema=schema, table=table))
        finally:
            cs.close()
            ctx.close()

    ## Prepare the engine
    engine = create_snowflake_engine(username, password, account, database, schema)

    ## Save to mssql table
    df.to_sql(name=table, con=engine, schema=schema, if_exists='append', chunksize=5000, index=False)

#def to_table(df, table, username, password, account, database, schema, create=False, truncate=False):
#    """
#    Function to append a DataFrame onto an existing mssql table.
#
#    Parameters
#    ----------
#    df : DataFrame
#        DataFrame to be saved. The DataFrame column/index names must match those on the mssql table exactly.
#    table : str
#        The specific table within the database. e.g.: 'LowFlowSiteRestrictionDaily'
#    username : str
#        The username
#    password : str
#        The password
#    account : str
#        account is the name assigned to your account by Snowflake. In the hostname you received from Snowflake (after your account was provisioned), your account name is the full/entire string to the left of snowflakecomputing.com
#    database : str
#        The specific database within the server. e.g.: 'LowFlows'
#    schema : str
#        The schema associated with the table to be read.
#
#    Returns
#    -------
#    None
#    """
#    today1 = pd.Timestamp.today().strftime('%Y%m%dT%H%M%S')
#
#    file_name = '{table}_{date}.csv'.format(table=table, date=today1)
#    file_name = '{table}_{date}.parquet'.format(table=table, date=today1)
#    file_path = os.path.abspath(file_name)
##    df.to_csv(file_path, index=False, header=False)
#    df.to_parquet(file_path, allow_truncated_timestamps=True)
#
#    engine = create_snowflake_engine(username, password, account, database, schema)
#
#    with engine.connect() as con:
#
#        if create:
#            df.head(0).to_sql(name=table,
#                              con=con,
#                              if_exists="replace",
#                              index=False)
#
#    ctx = snowflake.connector.connect(
#        user=username,
#        password=password,
#        account=account,
#        database=database,
#        schema=schema,
#        )
#    cs = ctx.cursor()
#    try:
#        if truncate:
#            cs.execute("truncate table {table}".format(table=table))
#        cs.execute("put file://{file_path}* @%{table}".format(file_path=file_path, table=table))
#        cs.execute("copy into {table} file_format = (type = parquet)".format(table=table))
#    finally:
#        cs.close()
#        ctx.close()
#
#    if truncate:
#        con.execute(f"truncate table {table}")
#
#    con.execute("put file://{file_path}* @%{table}".format(file_path=file_path, table=table))
#    con.execute("copy into {table}".format(table=table))

#def del_table_rows(username, password, account, database, schema, table=None, pk_df=None, stmt=None):
#    """
#    Function to selectively delete rows from an mssql table.
#
#    Parameters
#    ----------
#    server : str
#        The server name. e.g.: 'SQL2012PROD03'
#    database : str
#        The specific database within the server. e.g.: 'LowFlows'
#    table : str or None if stmt is a str
#        The specific table within the database. e.g.: 'LowFlowSiteRestrictionDaily'
#    pk_df : DataFrame
#        A DataFrame of the primary keys of the table for the rows that should be removed.
#    stmt : str
#        SQL delete statement. Will override everything except server and database.
#
#    Returns
#    -------
#    None
#
#    Notes
#    -----
#    Using the pk_df is the only way to ensure that specific rows will be deleted from composite keys. The column data types and names of pk_df must match the equivelant columns in the SQL table. The procedure creates a temporary table from the pk_df then deletes the rows in the target table based on the temp table. Then finally deletes the temp table.
#    """
#
#    ### Make the delete statement
##    del_where_list = sql_where_stmts(**kwargs)
#    if isinstance(stmt, str):
#        del_rows_stmt = stmt
#    elif isinstance(pk_df, pd.DataFrame):
#        temp_tab = '#temp_del_tab1'
#
#        ### Check the primary keys and unique keys
#        if '.' in table:
#            table1 = table.split('.')[1]
#        else:
#            table1 = table
#        pk_stmt = get_pk_stmt.format(db=database, table=table1)
#        pk = read_table(server, database, stmt=pk_stmt, username=username, password=password).name
#
#        un_stmt = get_un_stmt.format(db=database, table=table1)
#        un = read_table(server, database, stmt=un_stmt, username=username, password=password).name
#
#        if pk.empty:
#            raise ValueError('SQL table has no primary key. Please set one up.')
#        if (not np.isin(pk, pk_df.columns.tolist()).all()) & (not np.isin(un, pk_df.columns.tolist()).all()):
#            raise ValueError('The primary or unique keys in the SQL table does not match up with the pk_df')
#
#        sel_t1 = "select * from " + temp_tab
#        cols = pk_df.columns.tolist()
#        tab_where = [table + '.' + i for i in cols]
#        t1_where = [temp_tab + '.' + i for i in cols]
#        where_list = [t1_where[i] + ' = ' + tab_where[i] for i in np.arange(len(cols))]
#        where_stmt = " where " + " and ".join(where_list)
#        exists_stmt = "(" + sel_t1 + where_stmt + ")"
#        del_rows_stmt = "DELETE FROM " + table + " where exists " + exists_stmt
##    elif isinstance(del_where_list, list):
##        del_rows_stmt = "DELETE FROM " + table + " WHERE " + " AND ".join(del_where_list)
#    else:
#        raise ValueError('Please specify pk_df or stmt')
#
#    ### Delete rows
#    engine = create_snowflake_engine(username, password, account, database, schema)
#    with engine.begin() as conn:
#        if isinstance(pk_df, pd.DataFrame):
#            pk_df.to_sql(name=temp_tab, con=conn, if_exists='replace', chunksize=1000)
#        conn.execute(del_rows_stmt)

#def update_table_rows(df, username, password, account, database, schema, table, on=None, append=True):
#    """
#    Function to update rows from an mssql table. SQL table must have a primary key and the primary key must be in the input DataFrame.
#
#    Parameters
#    ----------
#    df : DataFrame
#        DataFrame with data to be overwritten in SQL table.
#    server : str
#        The server name. e.g.: 'SQL2012PROD03'
#    database : str
#        The specific database within the server. e.g.: 'LowFlows'
#    table : str
#        The specific table within the database. e.g.: 'LowFlowSiteRestrictionDaily'
#    on : None, str, or list of str
#        The index by which the update should be applied on. If None, then it uses the existing primary key(s).
#    append : bool
#        Should new sites be appended to the table?
#
#    Returns
#    -------
#    None
#    """
#    ### Check the primary keys
#    if on is None:
#        pk_stmt = get_pk_stmt.format(db=database, table=table)
#        pk = read_table(username, password, account, database, schema, pk_stmt).name.tolist()
#
#        if not pk:
#            raise ValueError('SQL table has no primary key. Please set one up or assign "on" explicitly.')
#        on = pk
#    elif isinstance(on, str):
#        on = [on]
#
#    ## Check that "on" are in the tables
#    df_bool = ~np.isin(on, df.columns).all()
#    if df_bool:
#        raise ValueError('"on" contains column names that are not in the df')
#
#
#    ### Make the update statement
#    temp_tab = '#temp_up1'
#    on_tab = [table + '.' + i for i in on]
#    on_temp = [temp_tab + '.' + i for i in on]
#    cols = df.columns.tolist()
#    val_cols = [i for i in cols if not i in on]
#    tab_list = [table + '.' + i for i in val_cols]
#    temp_list = [temp_tab + '.' + i for i in val_cols]
#    temp_list2 = [temp_tab + '.' + i for i in cols]
#    up_list = [tab_list[i] + ' = ' + temp_list[i] for i in np.arange(len(temp_list))]
#    on_list = [on_tab[i] + ' = ' + on_temp[i] for i in np.arange(len(on))]
##    up_stmt = "update " + table + " set " + ", ".join(up_list) + " from " + table + " inner join " + temp_tab + " on " + " and ".join(on_list)
#    if append:
#        up_stmt = "merge " + table + " using " + temp_tab + " on (" + " and ".join(on_list) + ") when matched then update set " + ", ".join(up_list) +  " WHEN NOT MATCHED BY TARGET THEN INSERT (" + ", ".join(cols) + ") values (" + ", ".join(temp_list2) + ");"
#    else:
#        up_stmt = "merge " + table + " using " + temp_tab + " on (" + " and ".join(on_list) + ") when matched then update set " + ", ".join(up_list) +  ";"
#
#    ### Run SQL code to update rows
#    engine = create_snowflake_engine(username, password, account, database, schema)
#    with engine.begin() as conn:
#        print('Saving data to temp table...')
#        df.to_sql(name=temp_tab, con=engine, if_exists='append', chunksize=5000, index=False)
#        print('Updating primary table...')
#        conn.execute(up_stmt)

max_stars_repo_name: claruspeter/pdsf

max_issues_repo_name: claruspeter/pdsf

max_forks_repo_name: claruspeter/pdsf


text:
# Script used to visualize 2d3ds dataset

import scipy.io as sio
import open3d as o3d
import colorsys
import numpy as np

def generate_semantic_color(area):
    category = []
    for space in area.Disjoint_Space:
        for object in space.object:
            category.append(object.name.split("_")[0])

    category = list(set(category))
    color_dict = {}

    for x in range(len(category)):
        HSV = [x * 1.0 / len(category), 0.5, 0.5]
        RGB = colorsys.hsv_to_rgb(*HSV)
        color_dict[category[x]] = RGB

    return color_dict

def visualize(raw_data, area, color, bbox=True, voxel=False):

    area = raw_data[area]
    area_points = []
    for space in area.Disjoint_Space:
        for object in space.object:
            points = object.points
            point_cloud = o3d.geometry.PointCloud()
            point_cloud.points = o3d.utility.Vector3dVector(points)

            # Semantic color
            if color == "semantic":
                color_dict = generate_semantic_color(area)
                point_cloud.paint_uniform_color(color_dict.get(object.name.split("_")[0]))
            else:
                # RGB color
                point_cloud.colors = o3d.utility.Vector3dVector(object.RGB_color / 255)

            if bbox:
                Bbox = np.reshape(object.Bbox, (6, 1))
                boundingBox = o3d.geometry.AxisAlignedBoundingBox(Bbox[:3], Bbox[3:])
                boundingBox.color = (1, 1, 1)
                area_points.append(boundingBox)

            # Voxelization
            if voxel:
                point_cloud = o3d.geometry.VoxelGrid.create_from_point_cloud(point_cloud,
                                                                         voxel_size=0.05)

            area_points.append(point_cloud)

    o3d.visualization.draw_geometries(area_points)

if __name__ == "__main__":
    raw_data = sio.loadmat('pointcloud.mat', squeeze_me=True, struct_as_record=False)
    visualize(raw_data, 'Area_3', 'semantic', bbox=True, voxel=False)

ext: py

reason: import numpy,import scipy


text:
# Reading JMA radar data in netcdf format
# take statistics for the whole Japanese region
import netCDF4
import matplotlib.pyplot as plt
import numpy as np
import h5py

import glob
import gzip
import subprocess
import sys
import os.path

# extract data from nc file
def ext_nc_JMA(fname):
    nc = netCDF4.Dataset(fname, 'r')
    eps = 0.001 # small number
    #
    # dimensions
    nx = len(nc.dimensions['LON'])
    ny = len(nc.dimensions['LAT'])
    nt = len(nc.dimensions['TIME'])
    print("dims:",nx,ny,nt)
    #
    # ext variable
    lons = nc.variables['LON'][:]
    lats = nc.variables['LAT'][:]
    R = nc.variables['PRATE'][:]
    #
    # clip area around tokyo
    lat_tokyo = 35.681167
    lon_tokyo = 139.767052
    nx_clip = 200
    ny_clip = 200
    i0=np.argmin(np.abs(lons.data-lon_tokyo)) - int(nx_clip/2)
    j0=np.argmin(np.abs(lats.data-lat_tokyo)) - int(ny_clip/2)
    i1=i0+nx_clip
    j1=j0+ny_clip
    # choose ii and jj to span the whole dataset area
    Rslct = np.zeros(35)
    count = 0
    for ii in range(-8,2+1):
        for jj in range(-8,7+1):
            ii0 = i0 + nx_clip*ii
            jj0 = j0 + ny_clip*jj
            ii1 = ii0 + nx_clip
            jj1 = jj0 + ny_clip
            # extract data
            Rclip = R.data[0,jj0:jj1,ii0:ii1]
            Rclip = Rclip.T   # transpose so that i index comes first
            lon_clip=lons.data[i0:i1]
            lat_clip=lats.data[j0:j1]
            # ratio of valid value
            rate_valid = 1- np.sum(Rclip < -eps)/Rclip.size
            if rate_valid > (1.0-eps):
                print(ii+8,jj+8,ii0,jj0,lons.data[ii0],lats.data[jj0],ii1,jj1,lons.data[ii1],lats.data[jj1],rate_valid,Rclip.min(),Rclip.max())
                Rslct[count] = Rclip.max()
                count = count + 1
                #print("rainfall min:max",Rclip.min(),Rclip.max())
    import pdb;pdb.set_trace()
    # save data
    return(Rslct)

# read
infile_root = '../data/jma_radar/2017/'
print('dir:',infile_root)

nx = 200
ny = 200
nt = 12

for infile in sorted(glob.iglob(infile_root + '/*/*/*00utc.nc.gz')):
    # read 1hour data at a time
    # initialize with -999.0
    R1h = np.full((nt,nx,ny),-999.0,dtype=np.float32)
    for i in range(12):
        shour = '{0:02d}utc'.format(i*5)
        in_zfile = infile.replace('00utc',shour)
        print('reading zipped file:',in_zfile)
        # '-k' option for avoiding removing gz file
        subprocess.run('gunzip -kf '+in_zfile,shell=True)
        in_nc=in_zfile.replace('.gz','')
        print('reading nc file:',in_nc)
        if os.path.exists(in_nc):
            Rclip = ext_nc_JMA(in_nc)
        else:
            print('nc file not found!!!',in_nc)
            next
        R1h[i,:,:]=Rclip
        subprocess.run('rm '+in_nc,shell=True)
    # write to h5 file
    h5fname = infile.split('/')[-1]
    h5fname = h5fname.replace('.nc.gz','.h5')
    print('writing h5 file:',h5fname)
    h5file = h5py.File('../data_h5/'+h5fname,'w')
    h5file.create_dataset('R',data= R1h)
    h5file.close()
    #sys.exit()

reason: import numpy


text:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import csv

#hhids=[86, 59, 77, 26, 93, 101, 114, 171, 1086, 1403]
# hhids=['1202', '871', '1103', '585', '59', '2755', '2233', '86', '114', '2575', '1700', '974', '1800',
#  '370', '187', '1169', '1718', '545', '94', '2018', '744', '2859', '2925', '484', '2953', '171', '2818', '1953',
#  '1697', '1463', '499', '1790', '1507', '1642', '93', '1632',
#  '1500', '2472', '2072', '2378', '1415', '2986', '1403', '2945', '77', '1792',
#  '624', '379', '2557', '890', '1192', '26', '2787', '2965', '2980', '434', '2829',
#  '503', '2532', '946', '2401', '1801','2337','1086','1714','1283','252','2814']

#compile
csvfile = open('violin/periodsb10b135.csv','w', newline='')
writer = csv.writer(csvfile, delimiter=',')
writer.writerow(["Payback period"," ","Policy","PV system’s size"])

table=pd.read_csv('period/panel2sb10b135.csv')
for i in range(68):
    print(i)

    year =  table['No solar no incentive MPC'][0:68].iloc[i]
    writer.writerow([year,"Without incentives","MPC","4.4kWp Battery size = 13.5KWh"])

    year =  table['No solar no incentive RBC'][0:68].iloc[i]
    writer.writerow([year,"Without incentives","RBC","4.4kWp Battery size = 13.5KWh"])

    year =  table['No solar with incentive MPC'][0:68].iloc[i]
    writer.writerow([year,"With incentives","MPC","4.4kWp Battery size = 13.5KWh"])

    year =  table['No solar with incentive RBC'][0:68].iloc[i]
    writer.writerow([year,"With incentives","RBC","4.4kWp Battery size = 13.5KWh"])

table=pd.read_csv('period/panel4sb10b135.csv')
for i in range(68):
    print(i)

    year =  table['No solar no incentive MPC'][0:68].iloc[i]
    writer.writerow([year,"Without incentives","MPC","8.8kWp Battery size = 13.5KWh"])

    year =  table['No solar no incentive RBC'][0:68].iloc[i]
    writer.writerow([year,"Without incentives","RBC","8.8kWp Battery size = 13.5KWh"])

    year =  table['No solar with incentive MPC'][0:68].iloc[i]
    writer.writerow([year,"With incentives","MPC","8.8kWp Battery size = 13.5KWh"])

    year =  table['No solar with incentive RBC'][0:68].iloc[i]
    writer.writerow([year,"With incentives","RBC","8.8kWp Battery size = 13.5KWh"])

csvfile.close()

years = pd.read_csv("violin/periodsb10b135.csv")
sns.set(style="whitegrid", palette="pastel", color_codes=True)
#plt.title('Battery size: 13.5 kWh')
ax = sns.catplot(x=" ", y="Payback period",hue="Policy",col="PV system’s size", kind="violin", data=years,palette="BuGn_r",scale="count",cut=0,bw=.2,legend=False)

plt.legend(title="Policy",loc='upper right')
#plt.ylim(12,28)
#plt.title('Battery size: 13.5 kWh',loc='center')
plt.savefig("violin/periodsb10b135.png")
plt.show()

max_stars_repo_name: irasus-technologies/EnergyBoost

max_issues_repo_name: irasus-technologies/EnergyBoost

max_forks_repo_name: irasus-technologies/EnergyBoost


text:
import numpy as np
from time import time

def mp_corr(
    nf,
    chn,
    nreg,
    lags,
    npix,
    nq,
    queue_in=None,
    queue_out=None,
    data=None,
    use_mp=True,
    verbose=False,
):
    """Multi-tau correlator."""

    def verbose0(reg):
        # some output for testing purposes
        print("\nn: ", n)
        print("reg: ", reg)
        # print("sst {}\nsr:    {}\nsl:    {}".format(sst, sr, sl))
        print("g2c: ", g2c)
        # print("datreg: ", datreg[reg])
        # print("corf: ", corf)

    def verbose1(reg):
        # some output for testing purposes
        print("\nn: ", n)
        print("reg: ", reg)
        print("sst {}\nsr:    {}\nsl:    {}".format(sst, sr, sl))
        print("srr {}\nsll {}".format(srr[reg], sll[reg]))
        print("g2c: ", g2c)
        # print("corf: ", corf)
        # print("datreg: ", datreg[reg])

    def running_mean(m, s, x, k):
        mt = 1.0 * m
        m += (x - m) / k
        s += (x - mt) * (x - m)

    def correlator(matrx):

        for qi in xnq:
            k = 0
            # update mean
            sst[qi][k] += matrx[qi].mean()

            # update register
            datreg[qi][k, n % chn] = matrx[qi].copy()
            if qi == 0:
                regc[k] += 1

            for l in range(chn - 1):
                if n >= l + 1:
                    corf[qi][l] += (
                        datreg[qi][k, n % chn] * datreg[qi][k, (n - l - 1) % chn]
                    )
                    srr[qi][l] += datreg[qi][k, n % chn]
                    sll[qi][l] += datreg[qi][k, (n - l - 1) % chn]
                    if qi == 0:
                        g2c[l] += 1

            # print(np.max(srr[0]), np.max(sll[0]))
            k += 1
            correlator2(k, qi)

    def correlator2(k, qi):
        if ((n + 1) % 2 ** k) == 0:
            datreg[qi][k, regc[k] % chn] = (
                datreg[qi][k - 1, (regc[k - 1] - 2) % chn]
                + datreg[qi][k - 1, (regc[k - 1] - 1) % chn]
            ) / 2.0
            if qi == 0:
                regc[k] += 1

        for c in range(chn2):
            l = (k + 1) * chn2 + c - 1
            nn = chn2 * 2 ** k + (c + 1) * 2 ** k
            # if qi == 0:
            #     print("(n,k,c,l,nn)=", n,k,c,l,nn)
            if (((n + 1) % 2 ** k) == (nn % 2 ** k)) and ((n + 1) >= nn):
                # if qi == 0:
                #     print('Compute g2')
                #     print("regc mod chn", regc[k]%chn)
                #     print("regc", regc[k])
                #     print(l, (g2c[l]+chn2+c)%chn, g2c[l]%chn)
                corf[qi][l] += (
                    datreg[qi][k, (g2c[l] + chn2 + c) % chn]
                    * datreg[qi][k, g2c[l] % chn]
                )
                srr[qi][l] += datreg[qi][k, (g2c[l] + chn2 + c) % chn]
                sll[qi][l] += datreg[qi][k, g2c[l] % chn]
                if qi == 0:
                    g2c[l] += 1

        if (k + 1) < nreg:  # and not (n + 1) % 2**(k + 1):
            k += 1
            correlator2(k, qi)

    # initialization of variables
    tcalc = time()
    chn2 = int(chn / 2)
    xnq = range(nq)

    # initialize counters
    nlag = len(lags)
    g2c = np.zeros(nlag, dtype=np.int32)
    regc = np.zeros(nreg, dtype=np.int32)
    sst = np.zeros((nq, nreg), dtype=np.float32)

    # initialize registers
    datreg = []
    sll = []
    srr = []
    corf = []
    for iq in xnq:
        datreg.append(np.zeros((nreg, chn, npix[iq]), dtype=np.float32))
        sll.append(np.zeros((nlag, npix[iq]), dtype=np.float32))
        srr.append(np.zeros((nlag, npix[iq]), dtype=np.float32))
        corf.append(np.zeros((nlag, npix[iq]), dtype=np.float32))

    # END of declaring and initializing variables####
    n = 0
    while n < nf:
        if use_mp:
            chunk = queue_in.get()
        else:
            chunk = data
        chunk_size = chunk[0].shape[0]
        ni = 0
        while ni < chunk_size:
            matr = [qchunk[ni] for qchunk in chunk]
            correlator(matr)
            ni += 1
            n += 1

    dcorf = []
    for qi in xnq:
        corf[qi] /= g2c[:, None]
        srr[qi] /= g2c[:, None]
        sll[qi] /= g2c[:, None]
        # if qi == 0:
        #     print(srr[qi], sll[qi])
        srr[qi] = np.ma.masked_invalid(srr[qi])
        sll[qi] = np.ma.masked_invalid(sll[qi])
        srr[qi] = np.mean(srr[qi], axis=1)  # / npix[qi]
        sll[qi] = np.mean(sll[qi], axis=1)  # / npix[qi]
        # print(corf[qi].shape, npix[qi], srr)
        corf[qi] = corf[qi] / (srr[qi][:,None] * sll[qi][:,None])
        dcorf.append(1 / np.sqrt(npix[qi]) * np.std(corf[qi], axis=1))
        corf[qi] = np.mean(corf[qi], axis=1)

    # print('corf: ', corf)
    # print('\nsr: ', sr)
    # print('\nsl: ', sl)
    # print('\ng2c: ',g2c)

    tcalc = time() - tcalc
    if use_mp:
        # END OF MAIN LOOP put results to output queue
        queue_in.close()
        queue_in.join_thread()
        queue_out.put([corf, dcorf, tcalc])
    else:
        return corf, dcorf, tcalc

max_stars_repo_name: reiserm/Xana

max_issues_repo_name: reiserm/Xana

max_forks_repo_name: reiserm/Xana


text:
import os

import gym
import enum
import typing

import numpy as np
from trifinger_simulation import trifingerpro_limits
from trifinger_simulation.tasks import move_cube_on_trajectory as task

class ActionType(enum.Enum):
    """Different action types that can be used to control the robot."""

    #: Use pure torque commands.  The action is a list of torques (one per
    #: joint) in this case.
    TORQUE = enum.auto()
    #: Use joint position commands.  The action is a list of angular joint
    #: positions (one per joint) in this case.  Internally a PD controller is
    #: executed for each action to determine the torques that are applied to
    #: the robot.
    POSITION = enum.auto()
    #: Use both torque and position commands.  In this case the action is a
    #: dictionary with keys "torque" and "position" which contain the
    #: corresponding lists of values (see above).  The torques resulting from
    #: the position controller are added to the torques in the action before
    #: applying them to the robot.
    TORQUE_AND_POSITION = enum.auto()

class BaseCubeTrajectoryEnv(gym.GoalEnv):
    """Gym environment for moving cubes with TriFingerPro."""

    def __init__(
        self,
        goal_trajectory: typing.Optional[task.Trajectory] = None,
        action_type: ActionType = ActionType.POSITION,
        step_size: int = 1,
    ):
        """Initialize.

        Args:
            goal_trajectory: Goal trajectory for the cube.  If ``None`` a new
                random trajectory is sampled upon reset.
            action_type: Specify which type of actions to use.
                See :class:`ActionType` for details.
            step_size:  Number of actual control steps to be performed in one
                call of step().
        """
        # Basic initialization
        # ====================

        if goal_trajectory is not None:
            task.validate_goal(goal_trajectory)
        self.goal = goal_trajectory

        self.action_type = action_type

        if step_size < 1:
            raise ValueError("step_size cannot be less than 1.")
        self.step_size = step_size

        # will be initialized in reset()
        self.platform = None

        # Create the action and observation spaces
        # ========================================

        robot_torque_space = gym.spaces.Box(
            low=trifingerpro_limits.robot_torque.low,
            high=trifingerpro_limits.robot_torque.high,
        )
        robot_position_space = gym.spaces.Box(
            low=trifingerpro_limits.robot_position.low,
            high=trifingerpro_limits.robot_position.high,
        )
        robot_velocity_space = gym.spaces.Box(
            low=trifingerpro_limits.robot_velocity.low,
            high=trifingerpro_limits.robot_velocity.high,
        )

        object_state_space = gym.spaces.Dict(
            {
                "position": gym.spaces.Box(
                    low=trifingerpro_limits.object_position.low,
                    high=trifingerpro_limits.object_position.high,
                ),
                "orientation": gym.spaces.Box(
                    low=trifingerpro_limits.object_orientation.low,
                    high=trifingerpro_limits.object_orientation.high,
                ),
            }
        )

        if self.action_type == ActionType.TORQUE:
            self.action_space = robot_torque_space
            self._initial_action = trifingerpro_limits.robot_torque.default
        elif self.action_type == ActionType.POSITION:
            self.action_space = robot_position_space
            self._initial_action = trifingerpro_limits.robot_position.default
        elif self.action_type == ActionType.TORQUE_AND_POSITION:
            self.action_space = gym.spaces.Dict(
                {
                    "torque": robot_torque_space,
                    "position": robot_position_space,
                }
            )
            self._initial_action = {
                "torque": trifingerpro_limits.robot_torque.default,
                "position": trifingerpro_limits.robot_position.default,
            }
        else:
            raise ValueError("Invalid action_type")

        self.observation_space = gym.spaces.Dict(
            {
                "robot_observation": gym.spaces.Dict(
                    {
                        "position": robot_position_space,
                        "velocity": robot_velocity_space,
                        "torque": robot_torque_space,
                    }
                ),
                "object_observation": gym.spaces.Dict(
                    {
                        "position": object_state_space["position"],
                        "orientation": object_state_space["orientation"],
                    }
                ),
                "action": self.action_space,
                "desired_goal": object_state_space["position"],
                "achieved_goal": object_state_space["position"],
            }
        )

    def compute_reward(
        self,
        achieved_goal: task.Position,
        desired_goal: task.Position,
        info: dict,
    ) -> float:
        """Compute the reward for the given achieved and desired goal.

        Args:
            achieved_goal: Current position of the object.
            desired_goal: Goal position of the current trajectory step.
            info: An info dictionary containing a field "time_index" which
                contains the time index of the achieved_goal.

        Returns:
            The reward that corresponds to the provided achieved goal w.r.t. to
            the desired goal. Note that the following should always hold true::

                ob, reward, done, info = env.step()
                assert reward == env.compute_reward(
                    ob['achieved_goal'],
                    ob['desired_goal'],
                    info,
                )
        """
        # This is just some sanity check to verify that the given desired_goal
        # actually matches with the active goal in the trajectory.
        active_goal = np.asarray(
            task.get_active_goal(
                self.info["trajectory"], self.info["time_index"]
            )
        )
        assert np.all(active_goal == desired_goal), "{}: {} != {}".format(
            info["time_index"], active_goal, desired_goal
        )

        return -task.evaluate_state(
            info["trajectory"], info["time_index"], achieved_goal
        )

    def step(self, action):
        """Run one timestep of the environment's dynamics.

        When end of episode is reached, you are responsible for calling
        ``reset()`` to reset this environment's state.

        Args:
            action: An action provided by the agent (depends on the selected
                :class:`ActionType`).

        Returns:
            tuple:

            - observation (dict): agent's observation of the current
              environment.
            - reward (float): amount of reward returned after previous action.
            - done (bool): whether the episode has ended, in which case further
              step() calls will return undefined results.
            - info (dict): info dictionary containing the current time index.
        """
        raise NotImplementedError()

    def reset(self):
        raise NotImplementedError()

    def seed(self, seed=None):
        """Sets the seed for this env’s random number generator.

        .. note::

           Spaces need to be seeded separately.  E.g. if you want to sample
           actions directly from the action space using
           ``env.action_space.sample()`` you can set a seed there using
           ``env.action_space.seed()``.

        Returns:
            List of seeds used by this environment.  This environment only uses
            a single seed, so the list contains only one element.
        """
        self.np_random, seed = gym.utils.seeding.np_random(seed)
        task.seed(seed)
        return [seed]

    # def _create_observation(self, t, action):
    #     robot_observation = self.platform.get_robot_observation(t)
    #     camera_observation = self.platform.get_camera_observation(t)
    #     object_observation = camera_observation.filtered_object_pose
    #
    #     active_goal = np.asarray(
    #         task.get_active_goal(self.info["trajectory"], t)
    #     )
    #
    #     observation = {
    #         "robot_observation": {
    #             "position": robot_observation.position,
    #             "velocity": robot_observation.velocity,
    #             "torque": robot_observation.torque,
    #         },
    #         "object_observation": {
    #             "position": object_observation.position,
    #             "orientation": object_observation.orientation,
    #         },
    #         "action": action,
    #         "desired_goal": active_goal,
    #         "achieved_goal": object_observation.position,
    #     }
    #     return observation

    def _gym_action_to_robot_action(self, gym_action):
        # construct robot action depending on action type
        if self.action_type == ActionType.TORQUE:
            robot_action = self.platform.Action(torque=gym_action)
        elif self.action_type == ActionType.POSITION:
            robot_action = self.platform.Action(position=gym_action)
        elif self.action_type == ActionType.TORQUE_AND_POSITION:
            robot_action = self.platform.Action(
                torque=gym_action["torque"], position=gym_action["position"]
            )
        else:
            raise ValueError("Invalid action_type")

        return robot_action

    def _set_urdf_path(self):
        """
        Sets the paths for the URDFs to use depending upon the finger type
        """
        try:
            from ament_index_python.packages import get_package_share_directory

            self.robot_properties_path = get_package_share_directory(
                "robot_properties_fingers"
            )
        except Exception:
            self.robot_properties_path = os.path.join(
                os.path.dirname(__file__), "robot_properties_fingers"
            )
        from trifinger_simulation import finger_types_data
        print(self.robot_properties_path)
        urdf_file = finger_types_data.get_finger_urdf("trifingerpro")
        self.finger_urdf_path = os.path.join(
            self.robot_properties_path, "urdf", urdf_file
        )


text:
import argparse
from sklearn.externals import joblib
import numpy as np
import pandas as pd
from pymongo import MongoClient

from repository.cliente_repository import ClienteRepositoryMongo
from repository.produto_repository import ProdutoRepositoryMongo
from repository.faturamento_repository import FaturamentoRepositoryMongo
from repository.oportunidade_repository import OportunidadeRepositoryMongo
from repository.mongo_production_repository import MongoProductionRepository
from repository.mongo_repository import GenericMongoRepository

parser = argparse.ArgumentParser(description='Calculate eSocial Campaigns')

parser.add_argument('--pickle-filename', help='specify the pickled filename relative to "classify_data.py"', type=str,
                    default='user_user_cf_knn-10-11-2016-9-27-0.pkl')
parser.add_argument('--mongo-database-url', help='mongo database url', default='localhost')
parser.add_argument('--mongo-database-name', help='mongo database name', default='testando')
parser.add_argument('--customer-collection-name', help='mongo customer collection name', default='clientes')
parser.add_argument('--item-collection-name', help='mongo item collection name', default='produtos')
parser.add_argument('--billing-collection-name', help='mongo billing collection name', default='faturamentos')
parser.add_argument('--opportunity-collection-name', help='mongo opportunity collection name', default='oportunidades')

args = parser.parse_args()

user_user_cf = joblib.load(args.pickle_filename)

target_items = ['S-1-100201', 'S-1-090235']
db = MongoClient()['testando']

item_repository = ProdutoRepositoryMongo(MongoProductionRepository(args), args.item_collection_name)

columns = item_repository.get_items_code()

def load_data(cf_matrix):
    billing_repository = FaturamentoRepositoryMongo(MongoProductionRepository(args), args.billing_collection_name)
    count = billing_repository.count()

    for customer_code in cf_matrix.index:
        documents = billing_repository.find(0, count, {'cliente.codigo': customer_code})

        for doc in documents:
            item_code = doc.get('produto').get('codigo')
            cf_matrix.loc[customer_code][item_code] = 1

def init_matrix():
    opportunity_repository = OportunidadeRepositoryMongo(MongoProductionRepository(args),
                                                         args.opportunity_collection_name)
    customers = opportunity_repository.get_customers_code()

    data = []

    for customer in customers:
        target_features = [i * 0 for i in xrange(len(columns))]
        data.append(target_features)

    test_matrix = pd.DataFrame(data=data, index=customers, columns=columns, dtype=int)

    load_data(test_matrix)

    return test_matrix

def save_predictions(test_matrix):
    for customer_code in test_matrix.index:
        target_features = test_matrix.loc[customer_code].values
        predictions = user_user_cf.predict(target_features)

        for i in xrange(len(predictions)):
            recommendation = {'buy': predictions[i], 'codigoProduto': columns[i], 'codigoCliente': customer_code,
                              'version': args.pickle_filename}
            db.knnPredictions.save(recommendation)

test_matrix = init_matrix()
save_predictions(test_matrix)

max_stars_repo_name: SeniorSA/hybrid-rs-trainner

max_issues_repo_name: SeniorSA/recomenda

max_forks_repo_name: SeniorSA/recomenda


text:
from typing import Optional, Tuple

import numpy as np
from sklearn import datasets
from torch.utils.data import DataLoader, Dataset
import torch

# import torch.multiprocessing as multiprocessing

# multiprocessing.set_start_method("spawn")
# class DensityDataset:
#     def __init__(self, data, dtype=np.float32):
#         self.data = data
#         self.dtype = dtype

#     def __len__(self):
#         return len(self.data)

#     def __getitem__(self, idx: int) -> np.ndarray:
#         data = self.data[idx]
#         return np.np.ndarray(data, dtype=self.dtype)

def add_dataset_args(parser):
    parser.add_argument(
        "--seed",
        type=int,
        default=123,
        help="number of data points for training",
    )
    parser.add_argument(
        "--dataset",
        type=str,
        default="noisysine",
        help="Dataset to be used for visualization",
    )
    parser.add_argument(
        "--n_samples",
        type=int,
        default=5_000,
        help="number of data points for training",
    )
    parser.add_argument(
        "--noise",
        type=float,
        default=0.1,
        help="number of data points for training",
    )
    parser.add_argument(
        "--n_train",
        type=int,
        default=2_000,
        help="number of data points for training",
    )
    parser.add_argument(
        "--n_valid",
        type=int,
        default=1_000,
        help="number of data points for training",
    )
    return parser

class DensityDataset(Dataset):
    def __init__(self, n_samples: int = 10_000, noise: float = 0.1, seed: int = 123):
        self.n_samples = n_samples
        self.seed = seed
        self.noise = noise
        self.reset()

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx: int) -> torch.Tensor:
        data = self.data[idx]
        return data

    def reset(self):
        self._create_data()

    def _create_data(self):
        raise NotImplementedError

class GenericDataset(DensityDataset):
    def __init__(self, data):
        self.data = data

def get_data(
    N: int = 30,
    input_noise: float = 0.15,
    output_noise: float = 0.15,
    N_test: int = 400,
) -> Tuple[np.ndnp.ndarray, np.ndnp.ndarray, np.ndnp.ndarray]:
    np.random.seed(0)
    X = np.linspace(-1, 1, N)
    Y = X + 0.2 * np.power(X, 3.0) + 0.5 * np.power(0.5 + X, 2.0) * np.sin(4.0 * X)
    Y += output_noise * np.random.randn(N)
    Y -= np.mean(Y)
    Y /= np.std(Y)

    X += input_noise * np.random.randn(N)

    assert X.shape == (N,)
    assert Y.shape == (N,)

    X_test = np.linspace(-1.2, 1.2, N_test)

    return X[:, None], Y[:, None], X_test[:, None]

class SCurveDataset(DensityDataset):
    def _create_data(self):
        data, _ = datasets.make_s_curve(
            n_samples=self.n_samples, noise=self.noise, random_state=self.seed
        )
        data = data[:, [0, 2]]
        self.data = data

class BlobsDataset(DensityDataset):
    def _create_data(self):
        data, _ = datasets.make_blobs(n_samples=self.n_samples, random_state=self.seed)
        self.data = data

class MoonsDataset(DensityDataset):
    def _create_data(self):
        data, _ = datasets.make_moons(
            n_samples=self.n_samples, noise=self.noise, random_state=self.seed
        )
        self.data = data

class SwissRollDataset(DensityDataset):
    def _create_data(self):
        data, _ = datasets.make_swiss_roll(
            n_samples=self.n_samples, noise=self.noise, random_state=self.seed
        )
        data = data[:, [0, 2]]
        self.data = data

class NoisySineDataset(DensityDataset):
    def _create_data(self):

        rng = np.random.RandomState(seed=self.seed)
        x = np.abs(2 * rng.randn(1, self.n_samples))
        y = np.sin(x) + 0.25 * rng.randn(1, self.n_samples)
        self.data = np.vstack((x, y)).T

class CheckBoard(DensityDataset):
    def _create_data(self):
        rng = np.random.RandomState(self.seed)
        x1 = rng.rand(self.n_samples) * 4 - 2
        x2_ = rng.rand(self.n_samples) - rng.randint(0, 2, self.n_samples) * 2
        x2 = x2_ + (np.floor(x1) % 2)
        data = np.concatenate([x1[:, None], x2[:, None]], 1) * 2
        self.data = data + 0.001 * rng.randn(*data.shape)

max_stars_repo_name: IPL-UV/gaussflow

max_issues_repo_name: IPL-UV/gaussflow

max_forks_repo_name: IPL-UV/gaussflow


text:
import io
import time

import numpy as numpy
import edgetpu.classification.engine

import cv2
from config import Config

class VideoCamera(object):
    def __init__(self):
        with open(Config.LABEL_PATH, 'r', encoding="utf-8") as f:
            pairs = (l.strip().split(maxsplit=1) for l in f.readlines())
            self.labels = dict((int(k), v) for k, v in pairs)
        self.engine = edgetpu.classification.engine.ClassificationEngine(Config.MODEL_PATH)

        # Using OpenCV to capture from device 0. If you have trouble capturing
        # from a webcam, comment the line below out and use a video file
        # instead.
        self.video = cv2.VideoCapture(0)
        if self.video:
            self.video.set(3, 640)
            self.video.set(4, 480)
        # If you decide to use video.mp4, you must have this file in the folder
        # as the main.py.
        # self.video = cv2.VideoCapture('video.mp4')

    def __del__(self):
        print('closing camera')
        self.video.release()

    def get_frame(self):
        font                   = cv2.FONT_HERSHEY_SIMPLEX
        bottomLeftCornerOfText = (10, 470)
        fontScale              = 0.6
        fontColor              = (255,255,255)
        lineType               = 2

        annotate_text = ""

        _, width, height, channels = self.engine.get_input_tensor_shape()
        if not self.video.isOpened():
            print('Camera is not opened')
        ret, img = self.video.read()
        if not ret:
            print('Camera is not read')
        input = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
        input = cv2.resize(input, (width, height))
        input = input.reshape((width * height * channels))
        start_ms = time.time()
        results = self.engine.ClassifyWithInputTensor(input, top_k=Config.TOP_K)
        elapsed_ms = time.time() - start_ms
        # if results:
        #     print( "%s %.2f\n%.2fms" % (self.labels[results[0][0]], results[0][1], elapsed_ms*1000.0))
        if results and\
                    results[0][1] > Config.DETECT_THRESHOLD:
            annotate_text = "%s %.2f  %.2fms" % (
                self.labels[results[0][0]], results[0][1], elapsed_ms*1000.0)

            cv2.putText(img, annotate_text,
                bottomLeftCornerOfText,
                font,
                fontScale,
                fontColor,
                lineType)
        # We are using Motion JPEG, but OpenCV defaults to capture raw images,
        # so we must encode it into JPEG in order to correctly display the
        # video stream.
        ret, jpeg = cv2.imencode('.jpg', img)
        return jpeg.tobytes()


text:
import numpy as np
import tensorflow as tf

from elbow import Model, Gaussian, BernoulliMatrix, BetaMatrix
from elbow.models.factorizations import NoisyLatentFeatures

def latent_feature_model():
    K = 3
    D = 10
    N = 100

    a, b = np.float32(1.0), np.float32(1.0)

    pi = BetaMatrix(alpha=a, beta=b, shape=(K,), name="pi")
    B = BernoulliMatrix(p=pi, shape=(N, K), name="B")
    G = Gaussian(mean=0.0, std=1.0, shape=(K, D), name="G")
    D = NoisyLatentFeatures(B=B, G=G, std=0.1, name="D")

    jm = Model(D)

    return jm

def main():
    jm = latent_feature_model()

    sampled = jm.sample()
    jm["D"].observe(sampled["D"])

    sB = sampled["B"]
    sG = sampled["G"]
    sD = np.dot(sB, sG)
    mean_abs_err = np.mean(np.abs(sD - sampled["D"]))
    print "true latent values reconstruct observations with mean deviation %.3f" % (mean_abs_err)

    jm.train()
    posterior = jm.posterior()

    sess = jm.get_session()
    qB = sess.run(jm["B"].q_distribution().p)

    qG = posterior["q_G"]["mean"]
    qD = np.dot(qB, qG)
    mean_abs_err_inferred = np.mean(np.abs(qD - sampled["D"]))
    print "inferred latent values reconstruct observations with mean deviation %.3f" % (mean_abs_err_inferred)

if __name__ == "__main__":
    main()

max_stars_repo_name: davmre/elbow

max_issues_repo_name: davmre/bayesflow

max_forks_repo_name: davmre/elbow


text:
import sklearn
from sklearn import preprocessing
from sklearn.impute import SimpleImputer
from sklearn.decomposition import PCA
import pandas as pd
import numpy as np

#replace question marks with np.nan type
def replace_question_marks(df):
    try:
        df = df.replace({'?' : np.nan})
        print("Replaced all '?' to np.nan")
    except:
        print('No question marks found')
    return df

#check whether dataset is balanced
def check_class_distribution(df):
    print('Class distributions:')
    print(df.iloc[:,-1].value_counts())

#PCA dimension reducetion
def dimension_reduction(x_train, x_test, upper_bound=500, n_components=50,):
    if x_train.shape[1] >= upper_bound:
        pca = PCA(n_components=n_components, random_state=33)
        pca.fit(x_train)
        x_train= pd.DataFrame(pca.transform(x_train))
        x_test = pd.DataFrame(pca.transform(x_test))
        print("Reducing dimension form %s to %s"%(x_train.shape[1],n_components))
    return x_train, x_test

#encoder for X and y string values
def encode_labels(x_train, x_test, index=None):
    label_encoder = sklearn.preprocessing.LabelEncoder()
    df = pd.concat([x_train,x_test],axis=0)

    #encoding y labels
    if index == -1:
        print('Encoding y label values')
        not_null_df = df[df.notnull()]
        label_encoder.fit(not_null_df)
        x_train = label_encoder.transform(x_train)
        x_test = label_encoder.transform(x_test)

    #encoding x features
    else:
        print('Encoding X features')
        for i,t in enumerate(df.dtypes):
            if t == 'object':
                s_df = df.iloc[:,i]
                not_null_df = s_df.loc[s_df.notnull()]
                label_encoder.fit(not_null_df)
                try:
                    x_train.iloc[:,i] = x_train.iloc[:,i].astype('float')
                except:
                    x_train.iloc[:,i] = x_train.iloc[:,i].apply(lambda x: label_encoder.transform([x])[0] if x not in [np.nan] else x)
                try:
                    x_test.iloc[:,i] = x_test.iloc[:,i].astype('float')
                except:
                    x_test.iloc[:,i] = x_test.iloc[:,i].apply(lambda x: label_encoder.transform([x])[0] if x not in [np.nan] else x) #np.nan
    return x_train, x_test

#put class colunmn at end of dataframe
def reorder_columns(dataFrame):
    cols = dataFrame.columns.tolist()
    cols = cols[1:] + cols[:1]
    return dataFrame[cols]

#impute np.nan using given strategy
def impute_value(x_train, x_test, strategy):
    if strategy == None:
        return x_train.dropna(), x_test.dropna()
    else:
        imp = SimpleImputer(missing_values=np.nan, strategy=strategy)
        train_type_dic = dict()#keep original train data type before impute
        for i,t in enumerate(x_train.dtypes):
            if t != 'object':
                train_type_dic[i] = t
        test_type_dic = dict()#keep original test data type before impute
        for i,t in enumerate(x_test.dtypes):
            if t != 'object':
                test_type_dic[i] = t
        x_train = pd.DataFrame(imp.fit_transform(x_train))
        x_test = pd.DataFrame(imp.transform(x_test))
#         for key in train_type_dic:
#             x_train.iloc[:,key] = x_train.iloc[:,key].astype(train_type_dic[key])
#         for key in test_type_dic:
#             x_test.iloc[:,key] = x_test.iloc[:,key].astype(test_type_dic[key])
    return x_train, x_test

# default normalizer -> MinMaxScaelr
def normalize_data(X_train, X_test, scaler = preprocessing.MinMaxScaler()):
#     scaler = preprocessing.StandardScaler().fit(X_train)
    print('Normalized data by scaler: %s'%type(scaler))
    scaler = scaler.fit(X_train)
    X_train = scaler.transform(X_train)
    X_test = scaler.transform(X_test)
    return X_train, X_test


text:
##########################################################################
# NSAp - Copyright (C) CEA, 2015
# Distributed under the terms of the CeCILL-B license, as published by
# the CEA-CNRS-INRIA. Refer to the LICENSE file or to
# http://www.cecill.info/licences/Licence_CeCILL-B_V1-en.html for details.
##########################################################################

"""
Wrapper to Connectomist's 'DWI & Q-space' tab.
"""

# System import
import os
import shutil
import numpy as np
import nibabel
import warnings

# pyConnectomist import
from pyconnectomist import DEFAULT_CONNECTOMIST_PATH
from pyconnectomist.manufacturers import MANUFACTURERS
from pyconnectomist.exceptions import ConnectomistError
from pyconnectomist.exceptions import ConnectomistBadManufacturerNameError
from pyconnectomist.exceptions import ConnectomistBadFileError
from pyconnectomist.wrappers import ConnectomistWrapper
from pyconnectomist.utils.dwitools import read_bvals_bvecs
from pyconnectomist.utils.filetools import ptk_nifti_to_gis

# Define axis mapping
AXIS = {
    "x": 0,
    "y": 1,
    "z": 2
}

def gather_and_format_input_files(
        outdir,
        dwis,
        bvals,
        bvecs,
        b0_magnitude=None,
        b0_phase=None):
    """ Gather all files needed to start the preprocessing in the right format
    (Gis format for images and B0 maps).

    Parameters
    ----------
    outdir: str
        path to directory where to gather all files.
    dwis: str
        path to input Nifti files to be preprocessed.
    bvals: str
        path to .bval files associated to Nifti.
    bvecs: str
        path to .bvec files associated to Nifti.
    b0_magnitude: str (optional, default None)
        path to B0 magnitude map, may also contain phase.
        Required only if you want to make fieldmap-based
        correction of susceptibility distortions.
    b0_phase: str (optional, default None)
        not required if phase is already contained in
        b0_magnitude or if you don't want to make fieldmap-based
        correction of susceptibility distortions.

    Returns
    -------
    raw_dwi_dir:
        Connectomist's import data directory with files in Gis format.
    """
    # Check that files exist
    files = dwis + bvals + bvecs
    if b0_magnitude is not None:
        files.append(b0_magnitude)
    if b0_phase is not None:
        files.append(b0_phase)
    for path in files:
        if not os.path.isfile(path):
            raise ConnectomistBadFileError(path)

    # Create the directory if not existing
    if not os.path.isdir(outdir):
        os.mkdir(outdir)

    # If there is only one b0 map file and if this file contains 2 volumes,
    # split it in 2 files: magnitude and phase, assuming the first one is
    # magnitude
    if b0_magnitude and not b0_phase:
        b0_maps = nibabel.load(b0_magnitude)
        if b0_maps.shape[-1] == 2:
            voxels = b0_maps.get_data()
            header = b0_maps.get_header()
            affine = b0_maps.get_affine()
            magnitude = nibabel.Nifti1Image(voxels[..., 0], affine, header)
            phase = nibabel.Nifti1Image(voxels[..., 1], affine, header)
            b0_magnitude = os.path.join(outdir, "b0_magnitude.nii.gz")
            b0_phase = os.path.join(outdir, "b0_phase.nii.gz")
            magnitude.to_filename(b0_magnitude)
            phase.to_filename(b0_phase)

    # Go through all DWI data that must be converted
    copied_dwis = []
    copied_bvals = []
    copied_bvecs = []
    nb_of_sequences = len(dwis)
    for index, (dwi, bval, bvec) in enumerate(zip(dwis, bvals, bvecs)):

        # One sequence name special case
        if nb_of_sequences == 1:
            index = ""

        # Convert Nifti to Gis
        copied_dwis.append(ptk_nifti_to_gis(
            dwi, os.path.join(outdir, "dwi{0}.ima".format(index))))

        # Copy bval and bvec files, with homogeneous names
        bval_copy = os.path.join(outdir, "dwi{0}.bval".format(index))
        bvec_copy = os.path.join(outdir, "dwi{0}.bvec".format(index))
        shutil.copyfile(bval, bval_copy)
        shutil.copyfile(bvec, bvec_copy)
        copied_bvals.append(bval_copy)
        copied_bvecs.append(bvec_copy)

    # Convert and rename B0 map(s) if they are given
    if b0_magnitude is not None:
        b0_magnitude = ptk_nifti_to_gis(
            b0_magnitude, os.path.join(outdir, "b0_magnitude.ima"))

    if b0_phase is not None:
        b0_phase = ptk_nifti_to_gis(
            b0_phase, os.path.join(outdir, "b0_phase.ima"))

    return copied_dwis, copied_bvals, copied_bvecs, b0_magnitude, b0_phase

def data_import_and_qspace_sampling(
        outdir,
        subject_id,
        dwis,
        bvals,
        bvecs,
        manufacturer,
        flipX=False,
        flipY=False,
        flipZ=False,
        invertX=True,
        invertY=False,
        invertZ=False,
        b0_magnitude=None,
        b0_phase=None,
        phase_axis="y",
        slice_axis="z",
        path_connectomist=DEFAULT_CONNECTOMIST_PATH):
    """ Wrapper to Connectomist's 'DWI & Q-space' tab.

    Parameters
    ----------
    outdir: str
        path to Connectomist's output directory.
    subject_id: str
        the subject code in study.
    dwis: list of str
        path to Nifti diffusion-weighted datasets.
    bvals: list of str
        path to .bval files associated to the Nifti.
    bvecs: list of str
        path to .bvec files associated to the Nifti.
    manufacturer: str
        name of the manufacturer (e.g. "Siemens", "GE", "Philips" or "Bruker").
    flipX, flipY, flipZ: bool (optional, default False)
        if True invert the x, y or z-axis of data.
    invertX: bool (optional, default True)
        if True invert x-axis of diffusion model.
    invertY, invertZ: bool (optional, default False)
        if True invert y or z-axis of diffusion model.
    b0_magnitude: str (optional, default None)
        path to the magnitude fieldmap (if fieldmap-based correction of
        susceptibility distortions is to be used).
    b0_phase: str (optional, default None)
        path to phase fieldmap (if fieldmap-based correction of susceptibility
        distortions is to be used).
    phase_axis: str (optional, default 'y')
        the acquistion phase axis 'x', 'y' or 'z'.
    slice_axis: str (optional, default 'z')
        the acquistion slice axis 'x', 'y' or 'z'.
    path_connectomist: str (optional)
        path to the Connectomist executable.

    Returns
    -------
    outdir: str
        path to Connectomist's output directory.
    """
    # Check input parameters
    for axis in (phase_axis, slice_axis):
        if axis not in AXIS:
            raise ValueError("Invalid axis '{0}'.".format(axis))
    if len(set(map(len, (dwis, bvecs, bvals)))) != 1:
        raise ValueError("The DWIs, BVECs and BVALs must have the same number "
                         "of elements.")

    # Create the Connectomist's import data directory
    dwis, bvals, bvecs, b0_magnitude, b0_phase = gather_and_format_input_files(
        outdir,
        dwis,
        bvals,
        bvecs,
        b0_magnitude,
        b0_phase)

    # Dict with all parameters for connectomist
    algorithm = "DWI-Data-Import-And-QSpace-Sampling"
    parameters_dict = {
        # Parameters are ordered as they appear in connectomist's GUI

        # ---------------------------------------------------------------------
        # Field: "Diffusion weighted-images"
        "fileNameDwi":  ";".join(dwis),  # "DW data"
        "sliceAxis":    AXIS[slice_axis],  # "Slice axis", default "Z-axis"
        "phaseAxis":    AXIS[phase_axis],  # "Phase axis", default "Y-axis"
        "manufacturer": None,

        # Subfield: "Advanced parameters"
        "flipAlongX":           2 if flipX else 0,  # "Flip data along x"
        "flipAlongY":           2 if flipY else 0,
        "flipAlongZ":           2 if flipZ else 0,
        "numberOfDiscarded":    0,  # "#discarded images at beginning"
        "numberOfT2":        None,  # "#T2"
        "numberOfRepetitions":  1,  # "#repetitions"
        # ---------------------------------------------------------------------
        # Field: "Rotation of field of view", default is identity matrix
        "qSpaceTransform_xx": 1.0,
        "qSpaceTransform_xy": 0.0,
        "qSpaceTransform_xz": 0.0,
        "qSpaceTransform_yx": 0.0,
        "qSpaceTransform_yy": 1.0,
        "qSpaceTransform_yz": 0.0,
        "qSpaceTransform_zx": 0.0,
        "qSpaceTransform_zy": 0.0,
        "qSpaceTransform_zz": 1.0,
        # ---------------------------------------------------------------------
        # Field: "Q-space sampling"
        "qSpaceSamplingType": 4,  # default "spherical single-shell custom"
        "qSpaceChoice5BValueFileNames": ";".join(bvals),
        "qSpaceChoice5BValueThreshold": 50,
        "qSpaceChoice5OrientationFileNames": ";".join(bvecs),

        # Apparently Connectomist uses 2 as True, and 0 as False.
        "invertXAxis": 2 if invertX else 0,
        "invertYAxis": 2 if invertY else 0,
        "invertZAxis": 2 if invertZ else 0,

        # In this field but not used/handled parameters
        "qSpaceChoice1MaximumBValue":       1000,  # case Cartesian
        "qSpaceChoice2BValue":              1000,
        "qSpaceChoice3BValue":              1000,
        "qSpaceChoice4BValue":              1000,
        "qSpaceChoice6BValues":               "",
        "qSpaceChoice7BValues":               "",
        "qSpaceChoice8BValues":               "",
        "qSpaceChoice9BValues":               "",
        "qSpaceChoice10BValues":              "",
        "qSpaceChoice11BValues":              "",
        "qSpaceChoice1NumberOfSteps":         11,
        "qSpaceChoice2NumberOfOrientations":   6,
        "qSpaceChoice3NumberOfOrientations":   6,
        "qSpaceChoice4NumberOfOrientations":   6,
        "qSpaceChoice6NumberOfOrientations":   6,
        "qSpaceChoice7NumberOfOrientations":   6,
        "qSpaceChoice8NumberOfOrientations":   6,
        "qSpaceChoice10NumberOfOrientations": "",
        "qSpaceChoice11NumberOfOrientations": "",
        "qSpaceChoice13OrientationFileNames": "",
        # ---------------------------------------------------------------------
        # Field: micro structure config"
        "gradientCharacteristicsFileNames": "",
        # ---------------------------------------------------------------------
        # Field: "Work directory"
        "outputWorkDirectory": outdir,
        # ---------------------------------------------------------------------
        # unknown parameter
        "_subjectName": subject_id,
    }

    # Map the manufacturer name with Connectomist convention
    if manufacturer not in MANUFACTURERS:
        raise ConnectomistBadManufacturerNameError(manufacturer)
    parameters_dict["manufacturer"] = MANUFACTURERS[manufacturer]

    # Read bvals and bvecs
    bvalues, bvectors, nb_shells, nb_nodiff = read_bvals_bvecs(bvals, bvecs)

    # Update Connectomist step description
    parameters_dict["numberOfT2"] = nb_nodiff
    if nb_shells == 1:
        # Spherical single-shell custom
        parameters_dict["qSpaceSamplingType"] = 4
    else:
        warnings.warn(
            "'{0}' shell model(s) not handled yet.".format(nb_shells))
        # Arbitrary shell
        parameters_dict["qSpaceSamplingType"] = 12
        parameters_dict["qSpaceChoice13BValueFileNames"] = ";".join(bvals)
        parameters_dict["qSpaceChoice13BValueThreshold"] = 50.0
        parameters_dict["qSpaceChoice13OrientationFileNames"] = ";".join(bvecs)

    # Call with Connectomist
    process = ConnectomistWrapper(path_connectomist)
    parameter_file = ConnectomistWrapper.create_parameter_file(
        algorithm, parameters_dict, outdir)
    process(algorithm, parameter_file, outdir)

    # When there are multiple volume or when there are multiple t2 (nodif)
    # volumes, Connectomist merges them
    # rewrite bvec, bval file accordingly (remove extra T2 values)
    if nb_nodiff > 1:
        bval = os.path.join(outdir, "dwi.bval")
        dw_indexes = np.where(bvalues >= 100)[0]
        new_bvals = np.concatenate(([0], bvalues[dw_indexes]))
        np.savetxt(bval, new_bvals)
        bvec = os.path.join(outdir, "dwi.bvec")
        new_bvecs = np.concatenate(
            ([[0], [0], [0]], bvectors.T[:, dw_indexes]), axis=1)
        np.savetxt(bvec, new_bvecs)

    return outdir

max_stars_repo_name: neurospin/pyconnectomist

max_issues_repo_name: neurospin/pyconnectomist

max_forks_repo_name: neurospin/pyconnectomist


text:
import pylab as pl
from scipy import io

# Adding Files and locations
froot = '/home/hari/Documents/PythonCodes/EfferentFFR/'
f331 = True
noisecarr = False
shorter = True
if f331:
    froot = froot + 'f331/'
    xlim = (100, 500)
    if shorter:
        condstemlist = ['signalOnly', 'simultaneousNoise',
                        'noise500ms_ahead', 'forwardMasking']
    else:
        condstemlist = ['signalOnly', 'simultaneousNoise',
                        'noise500ms_ahead', 'noiseOnly',
                        'forwardMasking']
else:
    xlim = (70, 250)
    condstemlist = ['signalOnly', 'simultaneousNoise',
                    'noise500ms_ahead', 'noiseOnly',
                    'forwardMasking']

if noisecarr and not f331:
    froot = froot + 'NoiseCarr/'
    condstemlist = ['signalOnly', 'simultaneousNoise',
                    'noise500ms_ahead', 'forwardMasking']

subj = 'I07'

fpath = froot + subj + '/'

# These are so that the generated files are organized better
respath = fpath + 'RES/'

pl.figure()
for k, cond in enumerate(condstemlist):
    fname = respath + subj + '_' + cond + '_results.mat'
    dat = io.loadmat(fname)
    f = dat['f'].squeeze()

    cpow = dat['cpow'].squeeze()
    cplv = dat['cplv'].squeeze()
    Sraw = dat['Sraw'].squeeze()

    # Plot PLV
    ax1 = pl.subplot(3, 1, 1)
    pl.plot(f, cpow, linewidth=2)
    pl.hold(True)
    pl.ylabel('Response Magnitude (uV^2)', fontsize=16)
    pl.title(' Subject ' + subj + ' Efferent FFR results')
    pl.legend(condstemlist)

    # Plot power
    ax2 = pl.subplot(3, 1, 2, sharex=ax1)
    pl.plot(f, cplv, linewidth=2)
    pl.hold(True)
    pl.ylabel('Phase locking value', fontsize=16)

    # Plot raw power
    ax3 = pl.subplot(3, 1, 3, sharex=ax1)
    ch = [3, 4, 25, 26, 30, 31]
    pl.plot(f, Sraw[ch, :].mean(axis=0), linewidth=2)
    pl.hold(True)
    pl.ylabel('Raw power', fontsize=16)
    pl.xlabel('Frequency (Hz)', fontsize=16)

pl.xlim(xlim)
pl.show()

max_stars_repo_name: haribharadwaj/codebasket

max_issues_repo_name: haribharadwaj/codebasket

max_forks_repo_name: haribharadwaj/codebasket

reason: from scipy


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

import datetime
import itertools
import copy
import joblib
import numpy
import scipy.sparse
import segment
import collections
import skimage.io
import features
import color_space

def _calc_adjacency_matrix(label_img, n_region):
    r = numpy.vstack([label_img[:, :-1].ravel(), label_img[:, 1:].ravel()])
    b = numpy.vstack([label_img[:-1, :].ravel(), label_img[1:, :].ravel()])
    t = numpy.hstack([r, b])
    A = scipy.sparse.coo_matrix((numpy.ones(t.shape[1]), (t[0], t[1])), shape=(n_region, n_region), dtype=bool).todense().getA()
    A = A | A.transpose()

    for i in range(n_region):
        A[i, i] = True

    dic = {i : {i} ^ set(numpy.flatnonzero(A[i])) for i in range(n_region)}

    Adjacency = collections.namedtuple('Adjacency', ['matrix', 'dictionary'])
    return Adjacency(matrix = A, dictionary = dic)

def _new_adjacency_dict(A, i, j, t):
    Ak = copy.deepcopy(A)
    Ak[t] = (Ak[i] | Ak[j]) - {i, j}
    del Ak[i], Ak[j]
    for (p, Q) in Ak.items():
        if i in Q or j in Q:
            Q -= {i, j}
            Q.add(t)

    return Ak

def _new_label_image(F, i, j, t):
    Fk = numpy.copy(F)
    Fk[Fk == i] = Fk[Fk == j] = t
    return Fk

def _build_initial_similarity_set(A0, feature_extractor):
    S = list()
    for (i, J) in A0.items():
        S += [(feature_extractor.similarity(i, j), (i, j)) for j in J if i < j]

    return sorted(S)

def _merge_similarity_set(feature_extractor, Ak, S, i, j, t):
    # remove entries which have i or j
    S = list(filter(lambda x: not(i in x[1] or j in x[1]), S))

    # calculate similarity between region t and its adjacencies
    St = [(feature_extractor.similarity(t, x), (t, x)) for x in Ak[t] if t < x] +\
         [(feature_extractor.similarity(x, t), (x, t)) for x in Ak[t] if x < t]

    return sorted(S + St)

def hierarchical_segmentation(I, k = 100, feature_mask = features.SimilarityMask(1, 1, 1, 1)):
    F0, n_region = segment.segment_label(I, 0.8, k, 100)
    adj_mat, A0 = _calc_adjacency_matrix(F0, n_region)
    feature_extractor = features.Features(I, F0, n_region)

    # stores list of regions sorted by their similarity
    S = _build_initial_similarity_set(A0, feature_extractor)

    # stores region label and its parent (empty if initial).
    R = {i : () for i in range(n_region)}

    A = [A0]    # stores adjacency relation for each step
    F = [F0]    # stores label image for each step

    # greedy hierarchical grouping loop
    while len(S):
        (s, (i, j)) = S.pop()
        t = feature_extractor.merge(i, j)

        # record merged region (larger region should come first)
        R[t] = (i, j) if feature_extractor.size[j] < feature_extractor.size[i] else (j, i)

        Ak = _new_adjacency_dict(A[-1], i, j, t)
        A.append(Ak)

        S = _merge_similarity_set(feature_extractor, Ak, S, i, j, t)

        F.append(_new_label_image(F[-1], i, j, t))

    # bounding boxes for each hierarchy
    L = feature_extractor.bbox

    return (R, F, L)

def _generate_regions(R, L):
    n_ini = sum(not parent for parent in R.values())
    n_all = len(R)

    regions = list()
    for label in R.keys():
        i = min(n_all - n_ini + 1, n_all - label)
        vi = numpy.random.rand() * i
        regions.append((vi, L[i]))

    return sorted(regions)

def _selective_search_one(I, color, k, mask):
    I_color = color_space.convert_color(I, color)
    (R, F, L) = hierarchical_segmentation(I_color, k, mask)
    return _generate_regions(R, L)

def selective_search(I, color_spaces = ['rgb'], ks = [100], feature_masks = [features.SimilarityMask(1, 1, 1, 1)], n_jobs = -1):
    parameters = itertools.product(color_spaces, ks, feature_masks)
    region_set = joblib.Parallel(n_jobs = n_jobs)(joblib.delayed(_selective_search_one)(I, color, k, mask) for (color, k, mask) in parameters)

    #flatten list of list of tuple to list of tuple
    regions = sum(region_set, [])
    return sorted(regions)


text:
# encoding: utf-8
"""
@author:  wuxin.wang
@contact: wuxin.wang@whu.edu.cn
"""
import cv2
import random
import numpy as np
import torch
from torch.utils.data import Dataset

class train_wheat(Dataset):

    def __init__(self, root, marking, image_ids, transforms=None, test=False):
        super().__init__()
        self.root = root
        self.image_ids = image_ids
        self.marking = marking
        self.transforms = transforms
        self.test = test

    def __getitem__(self, index: int):
        image_id = self.image_ids[index]
        p_ratio = random.random()
        if self.test or p_ratio > 0.6:
            image, boxes = self.load_image_and_boxes(index)
        else:
            if p_ratio < 0.2:
                image, boxes = self.load_mosaic_image_and_boxes(index)
            elif p_ratio < 0.4:
                image, boxes = self.load_image_and_bboxes_with_cutmix(index)
            else:
                image, boxes = self.load_mixup_image_and_boxes(index)

        # there is only one class
        labels = torch.ones((boxes.shape[0],), dtype=torch.int64)

        target = {}
        target['boxes'] = boxes
        target['labels'] = labels
        target['image_id'] = torch.tensor([index])

        if self.transforms:
            for i in range(10):
                sample = self.transforms(**{
                    'image': image,
                    'bboxes': target['boxes'],
                    'labels': labels
                })
                if len(sample['bboxes']) > 0:
                    image = sample['image']
                    target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)
                    # target['boxes'][:,[0,1,2,3]] = target['boxes'][:,[1,0,3,2]]  #yxyx: be warning
                    break

        return image, target, image_id

    def __len__(self) -> int:
        return self.image_ids.shape[0]

    def load_image_and_boxes(self, index):
        image_id = self.image_ids[index]
        image = cv2.imread(f'{self.root}/train/{image_id}.jpg', cv2.IMREAD_COLOR)
        image = cv2.resize(image,(1024,1024))
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)
        image /= 255.0
        records = self.marking[self.marking['image_id'] == image_id]
        boxes = records[['x', 'y', 'w', 'h']].values
        boxes[:, 2] = boxes[:, 0] + boxes[:, 2]
        boxes[:, 3] = boxes[:, 1] + boxes[:, 3]
        return image, boxes

    def load_mosaic_image_and_boxes(self, index, imsize=1024):
        """
        This implementation of mosaic author:  https://www.kaggle.com/nvnnghia
        Refactoring and adaptation: https://www.kaggle.com/shonenkov
        """
        w, h = imsize, imsize
        s = imsize // 2

        xc, yc = [int(random.uniform(imsize * 0.25, imsize * 0.75)) for _ in range(2)]  # center x, y
        indexes = [index] + [random.randint(0, self.image_ids.shape[0] - 1) for _ in range(3)]

        result_image = np.full((imsize, imsize, 3), 1, dtype=np.float32)
        result_boxes = []

        for i, index in enumerate(indexes):
            image, boxes = self.load_image_and_boxes(index)
            if i == 0:
                x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc  # xmin, ymin, xmax, ymax (large image)
                x1b, y1b, x2b, y2b = w - (x2a - x1a), h - (y2a - y1a), w, h  # xmin, ymin, xmax, ymax (small image)
            elif i == 1:  # top right
                x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc
                x1b, y1b, x2b, y2b = 0, h - (y2a - y1a), min(w, x2a - x1a), h
            elif i == 2:  # bottom left
                x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)
                x1b, y1b, x2b, y2b = w - (x2a - x1a), 0, max(xc, w), min(y2a - y1a, h)
            elif i == 3:  # bottom right
                x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)
                x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)
            result_image[y1a:y2a, x1a:x2a] = image[y1b:y2b, x1b:x2b]
            padw = x1a - x1b
            padh = y1a - y1b

            boxes[:, 0] += padw
            boxes[:, 1] += padh
            boxes[:, 2] += padw
            boxes[:, 3] += padh

            result_boxes.append(boxes)

        result_boxes = np.concatenate(result_boxes, 0)
        np.clip(result_boxes[:, 0:], 0, 2 * s, out=result_boxes[:, 0:])
        result_boxes = result_boxes.astype(np.int32)
        result_boxes = result_boxes[
            np.where((result_boxes[:, 2] - result_boxes[:, 0]) * (result_boxes[:, 3] - result_boxes[:, 1]) > 0)]
        return result_image, result_boxes

    def load_image_and_bboxes_with_cutmix(self, index):
        image, bboxes = self.load_image_and_boxes(index)
        image_to_be_mixed, bboxes_to_be_mixed = self.load_image_and_boxes(
            random.randint(0, self.image_ids.shape[0] - 1))

        image_size = image.shape[0]
        cutoff_x1, cutoff_y1 = [int(random.uniform(image_size * 0.0, image_size * 0.49)) for _ in range(2)]
        cutoff_x2, cutoff_y2 = [int(random.uniform(image_size * 0.5, image_size * 1.0)) for _ in range(2)]

        image_cutmix = image.copy()
        image_cutmix[cutoff_y1:cutoff_y2, cutoff_x1:cutoff_x2] = image_to_be_mixed[cutoff_y1:cutoff_y2,
                                                                 cutoff_x1:cutoff_x2]

        # Begin preparing bboxes_cutmix.
        # Case 1. Bounding boxes not intersect with cut off patch.
        bboxes_not_intersect = bboxes[np.concatenate((np.where(bboxes[:, 0] > cutoff_x2),
                                                      np.where(bboxes[:, 2] < cutoff_x1),
                                                      np.where(bboxes[:, 1] > cutoff_y2),
                                                      np.where(bboxes[:, 3] < cutoff_y1)), axis=None)]

        # Case 2. Bounding boxes intersect with cut off patch.
        bboxes_intersect = bboxes.copy()

        top_intersect = np.where((bboxes[:, 0] < cutoff_x2) &
                                 (bboxes[:, 2] > cutoff_x1) &
                                 (bboxes[:, 1] < cutoff_y2) &
                                 (bboxes[:, 3] > cutoff_y2))
        right_intersect = np.where((bboxes[:, 0] < cutoff_x2) &
                                   (bboxes[:, 2] > cutoff_x2) &
                                   (bboxes[:, 1] < cutoff_y2) &
                                   (bboxes[:, 3] > cutoff_y1))
        bottom_intersect = np.where((bboxes[:, 0] < cutoff_x2) &
                                    (bboxes[:, 2] > cutoff_x1) &
                                    (bboxes[:, 1] < cutoff_y1) &
                                    (bboxes[:, 3] > cutoff_y1))
        left_intersect = np.where((bboxes[:, 0] < cutoff_x1) &
                                  (bboxes[:, 2] > cutoff_x1) &
                                  (bboxes[:, 1] < cutoff_y2) &
                                  (bboxes[:, 3] > cutoff_y1))

        # Remove redundant indices. e.g. a bbox which intersects in both right and top.
        right_intersect = np.setdiff1d(right_intersect, top_intersect)
        right_intersect = np.setdiff1d(right_intersect, bottom_intersect)
        right_intersect = np.setdiff1d(right_intersect, left_intersect)
        bottom_intersect = np.setdiff1d(bottom_intersect, top_intersect)
        bottom_intersect = np.setdiff1d(bottom_intersect, left_intersect)
        left_intersect = np.setdiff1d(left_intersect, top_intersect)

        bboxes_intersect[:, 1][top_intersect] = cutoff_y2
        bboxes_intersect[:, 0][right_intersect] = cutoff_x2
        bboxes_intersect[:, 3][bottom_intersect] = cutoff_y1
        bboxes_intersect[:, 2][left_intersect] = cutoff_x1

        bboxes_intersect[:, 1][top_intersect] = cutoff_y2
        bboxes_intersect[:, 0][right_intersect] = cutoff_x2
        bboxes_intersect[:, 3][bottom_intersect] = cutoff_y1
        bboxes_intersect[:, 2][left_intersect] = cutoff_x1

        bboxes_intersect = bboxes_intersect[np.concatenate((top_intersect,
                                                            right_intersect,
                                                            bottom_intersect,
                                                            left_intersect), axis=None)]

        # Case 3. Bounding boxes inside cut off patch.
        bboxes_to_be_mixed[:, [0, 2]] = bboxes_to_be_mixed[:, [0, 2]].clip(min=cutoff_x1, max=cutoff_x2)
        bboxes_to_be_mixed[:, [1, 3]] = bboxes_to_be_mixed[:, [1, 3]].clip(min=cutoff_y1, max=cutoff_y2)

        # Integrate all those three cases.
        bboxes_cutmix = np.vstack((bboxes_not_intersect, bboxes_intersect, bboxes_to_be_mixed)).astype(int)
        bboxes_cutmix = bboxes_cutmix[np.where((bboxes_cutmix[:, 2] - bboxes_cutmix[:, 0]) \
                                               * (bboxes_cutmix[:, 3] - bboxes_cutmix[:, 1]) > 500)]
        # End preparing bboxes_cutmix.

        return image_cutmix, bboxes_cutmix

    def load_mixup_image_and_boxes(self, index):
        image, boxes = self.load_image_and_boxes(index)
        r_image, r_boxes = self.load_image_and_boxes(random.randint(0, self.image_ids.shape[0] - 1))
        return (image + r_image) / 2, np.vstack((boxes, r_boxes)).astype(np.int32)

max_stars_repo_name: ratthachat/wheatdetection

max_issues_repo_name: ratthachat/wheatdetection

max_forks_repo_name: ratthachat/wheatdetection


text:
""" Benchmark functions for fftpack.pseudo_diffs module
"""
from __future__ import division, absolute_import, print_function

import sys

from numpy import arange, sin, cos, pi, exp, tanh, sign
from scipy.fftpack import diff, fft, ifft, tilbert, hilbert, shift, fftfreq

def random(size):
    return rand(*size)

def direct_diff(x,k=1,period=None):
    fx = fft(x)
    n = len(fx)
    if period is None:
        period = 2*pi
    w = fftfreq(n)*2j*pi/period*n
    if k < 0:
        w = 1 / w**k
        w[0] = 0.0
    else:
        w = w**k
    if n > 2000:
        w[250:n-250] = 0.0
    return ifft(w*fx).real

def direct_tilbert(x,h=1,period=None):
    fx = fft(x)
    n = len(fx)
    if period is None:
        period = 2*pi
    w = fftfreq(n)*h*2*pi/period*n
    w[0] = 1
    w = 1j/tanh(w)
    w[0] = 0j
    return ifft(w*fx)

def direct_hilbert(x):
    fx = fft(x)
    n = len(fx)
    w = fftfreq(n)*n
    w = 1j*sign(w)
    return ifft(w*fx)

def direct_shift(x,a,period=None):
    n = len(x)
    if period is None:
        k = fftfreq(n)*1j*n
    else:
        k = fftfreq(n)*2j*pi/period*n
    return ifft(fft(x)*exp(k*a)).real

class Bench(object):
    params = [
        [100, 256, 512, 1000, 1024, 2048, 2048*2, 2048*4],
        ['fft', 'direct'],
    ]
    param_names = ['size', 'type']
    goal_time = 0.5

    def setup(self, size, type):
        size = int(size)

        x = arange(size)*2*pi/size
        a = 1
        self.a = a
        if size < 2000:
            self.f = sin(x)*cos(4*x)+exp(sin(3*x))
            self.sf = sin(x+a)*cos(4*(x+a))+exp(sin(3*(x+a)))
        else:
            self.f = sin(x)*cos(4*x)
            self.sf = sin(x+a)*cos(4*(x+a))

    def time_diff(self, size, soltype):
        if soltype == 'fft':
            diff(self.f, 3)
        else:
            direct_diff(self.f,3)

    def time_tilbert(self, size, soltype):
        if soltype == 'fft':
            tilbert(self.f, 1)
        else:
            direct_tilbert(self.f, 1)

    def time_hilbert(self, size, soltype):
        if soltype == 'fft':
            hilbert(self.f)
        else:
            direct_hilbert(self.f)

    def time_shift(self, size, soltype):
        if soltype == 'fft':
            shift(self.f, self.a)
        else:
            direct_shift(self.f, self.a)

max_stars_repo_name: sbellem/scipy

max_issues_repo_name: sbellem/scipy

max_forks_repo_name: sbellem/scipy

reason: from numpy,from scipy


text:
from numpy import arange, pi
import matplotlib.pyplot as plt

class Function:
    domain = arange(0, pi, 0.1)
    codomain = list()

    def __init__(self, func):
        self.calculate = func

    def build_plot(self):
        self.codomain = [self.calculate(x) for x in self.domain]
        plt.plot(self.domain, self.codomain, linewidth=2.0, label="Original function")

reason: from numpy


text:
import scipy
import numpy as np
import gp
import genepattern
from gp.data import _obtain_io, _apply_backwards_compatibility
import pandas as pd

def hellopipilworld():
    return u'Niltze Cemanahuac! (Hello "World"! in Nahuatl.)'

def list2cls(in_list, name_of_out='output.cls', sep='\t'):
    """This function creates a CLS file from a list-like object"""
    # print("~~~~~"+str(metadata_subset.shape)+'~~~~~')
    cls = open(name_of_out, 'w')
    cls.write("{}{}{}{}1\n".format(len(in_list), sep, len(np.unique(in_list)), sep))
    cls.write("#{}{}\n".format(sep, sep.join(np.unique(in_list).astype(str))))
    cls.write(sep.join(in_list.astype(str))+'\n')
    cls.close()

def df2gct(df, extra_columns=0, use_index=True, name='output.gct', add_dummy_descriptions=False):
    if add_dummy_descriptions:
        if add_dummy_descriptions not in list(df):
            df.insert(0, 'Description', df.index)
            extra_columns += 1
    f = open(name, 'w')
    f.write("#1.2\n")
    f.write("\t".join([str(df.shape[0]), str(df.shape[1] - extra_columns)]) + "\n")
    f.write(df.to_csv(sep='\t', index=use_index))
    f.close()
    return

def custom_pearson_corr(x, y):
    # [-1,1]
    return scipy.stats.pearsonr(x, y)[0]

def custom_pearson_dist(x, y):
    # [-1,1]
    return 1 - scipy.stats.pearsonr(x, y)[0]

def custom_spearman_corr(x, y):
    return scipy.stats.spearmanr(x, y)[0]

def custom_spearman_dist(x, y):
    return 1 - scipy.stats.spearmanr(x, y)[0]

def absolute_spearman(x, y):
    return abs(scipy.stats.spearmanr(x, y)[0])

def custom_kendall_tau_corr(x, y):
    return scipy.stats.kendalltau(x, y)[0]

def custom_kendall_tau_dist(x, y):
    return 1 - scipy.stats.kendalltau(x, y)[0]

def absolute_pearson_corr(x, y):
    return np.abs(scipy.stats.pearsonr(x, y)[0])

def absolute_pearson_dist(x, y):
    return 1 - np.abs(scipy.stats.pearsonr(x, y)[0])

def uncentered_pearson_corr(x, y):
    # [-1,1]
    if len(x) != len(y):
        # Uncentered Pearson Correlation cannot be computed for vectors of different length.
        print('Uncentered Pearson Correlation cannot be computed for vectors of different length.')
        return np.nan

    else:
        # Using the definition from eq (4) in https://www.biomedcentral.com/content/supplementary/1477-5956-9-30-S4.PDF
        # x_squared =
        # y_squared =
        return np.sum(np.multiply(x, y)) / (np.sqrt(np.sum(np.square(x))) * np.sqrt(np.sum(np.square(y))))

def uncentered_pearson_dist(x, y):
    # [0,2]
    if len(x) != len(y):
        # Uncentered Pearson Correlation cannot be computed for vectors of different length.
        print('Uncentered Pearson Correlation cannot be computed for vectors of different length.')
        return np.nan

    else:
        # Using the definition from eq (4) in https://www.biomedcentral.com/content/supplementary/1477-5956-9-30-S4.PDF
        # x_squared =
        # y_squared =
        return 1 - (np.sum(np.multiply(x, y)) / (np.sqrt(np.sum(np.square(x))) * np.sqrt(np.sum(np.square(y)))))

def absolute_uncentered_pearson_corr(x, y):
    if len(x) != len(y):
        # Uncentered Pearson Correlation cannot be computed for vectors of different length.
        print('Uncentered Pearson Correlation cannot be computed for vectors of different length.')
        return np.nan

    else:
        # Using the definition from eq (4) in https://www.biomedcentral.com/content/supplementary/1477-5956-9-30-S4.PDF
        return np.abs(np.sum(np.multiply(x, y)) / (np.sqrt(np.sum(np.square(x))) * np.sqrt(np.sum(np.square(y)))))

def absolute_uncentered_pearson_dist(x, y):
    if len(x) != len(y):
        # Uncentered Pearson Correlation cannot be computed for vectors of different length.
        print('Uncentered Pearson Correlation cannot be computed for vectors of different length.')
        return np.nan

    else:
        # Using the definition from eq (4) in https://www.biomedcentral.com/content/supplementary/1477-5956-9-30-S4.PDF
        return 1- (np.abs(np.sum(np.multiply(x, y)) / (np.sqrt(np.sum(np.square(x))) * np.sqrt(np.sum(np.square(y))))))

def mydist(p1, p2):
    # a custom function that just computes Euclidean distance
    diff = p1 - p2
    return np.vdot(diff, diff) ** 0.5

def custom_euclidean_dist(x, y):
    return scipy.spatial.distance.euclidean(x, y)

def custom_euclidean_sim(x, y):
    return 1/(1+scipy.spatial.distance.euclidean(x, y))

def dendodist(V, dist=mydist):
    dists = np.array([dist(a[0], a[1]) for a in V])
    return np.cumsum(dists)

def custom_manhattan_dist(x, y):
    return scipy.spatial.distance.cityblock(x, y)

def custom_manhattan_sim(x, y):
    return 1/(1+scipy.spatial.distance.cityblock(x, y))

def custom_cosine_dist(x, y):
    return scipy.spatial.distance.cosine(x, y)

def custom_cosine_sim(x, y):
    return 1-scipy.spatial.distance.cosine(x, y)

def makeODF(output_data, vals, file_name='noname.odf'):
    f = open(file_name, 'w')
    f.write("ODF 1.0\n")  # Hard-coding spance, not tab here.
    f.write("HeaderLines=18\n")  # hard-coding 19+2-3 lines here. Needs to change.
    # f.write("COLUMN_NAMES:Rank\tFeature\tDescription\tScore\tFeature P\tFeature P Low\tFeature P High\tFDR(BH)\tQ Value"
    #         "\tBonferroni\tmaxT\tFWER\tFold Change\tALL Mean\tALL Std\tAML Mean\tAML Std\tk\n")
    # f.write("COLUMN_TYPES:int\tString\tString\tdouble\tdouble\tdouble\tdouble\tdouble\tdouble\tdouble\tdouble\tdouble"
    #         "\tdouble\tdouble\tdouble\tdouble\tdouble\tint\n")

    f.write("COLUMN_NAMES:"+"\t".join(list(output_data))+"\n")
    # f.write("COLUMN_DESCRIPTIONS:"+"\t".join(list(output_data))+"\n")
    f.write("COLUMN_TYPES:"+"\t".join(['String', 'String', 'double', 'String'])+"\n")  # TODO: automate this.
    #
    # print(list(output_data))
    # f.write("COLUMN_DESCRIPTIONS:\n")
    # f.write("Model=DiffEx\n")
    f.write("Model=Comparative Marker Selection\n")
    f.write("Dataset File="+vals['gct']+"\n")
    f.write("Class File="+vals['cls']+"\n")
    f.write("Permutations="+str(vals['n_perm'])+"\n")
    # f.write("Balanced=false\n")  # TODO: remove this one?
    # f.write("Complete=false\n")  # TODO: remove this one?
    # f.write("Test Direction=2 Sided\n")  # TODO: remove this one?
    f.write("Class 0="+vals['class_0']+"\n")
    f.write("Class 1="+vals['class_1']+"\n")
    f.write("Test Statistic="+vals['func']+"\n")
    f.write("pi0=\n")
    f.write("lambda=\n")
    f.write("pi0(lambda)=\n")
    f.write("cubic spline(lambda)=\n")
    f.write("Random Seed="+str(vals['rand_seed'])+"\n")
    f.write("Smooth p-values=\n")
    f.write("DataLines="+str(vals['dat_lines'])+"\n")
    f.write("RowNamesColumn=0\n")
    f.write("RowDescriptionsColumn=1\n")
    f.write(output_data.to_csv(sep='\t', index=False, header=False))
    return

def get_file_from_server(gene_pattern_url, file_type='GCT'):
    file_io = gp.GPFile(genepattern.session.get(0),gene_pattern_url)
    # # get the input filename and job number
    # jobNum = gene_pattern_url.split("/")[-2]
    # input_file_Name = gene_pattern_url.split("/")[-1]
    #
    # # get the GenePattern input job object and my username
    # lastJob = gp.GPJob(genepattern.get_session(0), jobNum)
    # myUserId = genepattern.get_session(0).username
    #
    # # Handle all the various initialization types and get an IO object
    # file_io = _obtain_io(lastJob.get_file(input_file_Name))
    #
    if file_type == 'GCT':
        # Load the GCT file into a DataFrame
        data = pd.read_table(file_io.open(), header=2, index_col=0)
        # Apply backwards compatible methods
        _apply_backwards_compatibility(data)
    elif file_type == 'CLS':
        # Load the CLS into a **list**!
        temp = file_io.read().split('\n')
        data = [int(i) for i in temp[2].strip('\n').split(' ')]
    else:
        print("Unfortunatley, reading the file type {} is not supported at the moment :/ returning the file as a string".format(file_type))
        data = file_io.read()

    return data

## Additional IO from CCAL (12/25/21)
def read_gmt(gmt_file_path, drop_description=True):

    lines = []

    with open(gmt_file_path) as gmt_file:

        for line in gmt_file:

            split = line.strip().split(sep="\t")

            lines.append(split[:2] + [gene for gene in set(split[2:]) if gene])

    df = pd.DataFrame(lines)

    df.set_index(0, inplace=True)

    df.index.name = "Gene Set"

    if drop_description:

        df.drop(1, axis=1, inplace=True)

        df.columns = tuple("Gene {}".format(i) for i in range(0, df.shape[1]))

    else:

        df.columns = ("Description",) + tuple(
            "Gene {}".format(i) for i in range(0, df.shape[1] - 1)
        )

    return df

max_stars_repo_name: alex-wenzel/ccal-noir

max_issues_repo_name: alex-wenzel/ccal-noir

max_forks_repo_name: edjuaro/ccal-noir


text:
import warnings
warnings.filterwarnings("ignore")
import os
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"

#import gym
#import bandit
#from BPG.grf_bpg import BPG
#from BPG.grf_policy import Actor, Critic
#import numpy as np

from utils.execute import Executor

env_id = 'continuous-armed_bandit-v0'
algo_id = 'ppo'
train_timestep = int(1e4)
test_timestep = 20
algo_para = {'learning_rate': 2.5e-3, 'gamma': 0, 'ent_coef': 0.0}

executor = Executor(env_id, algo_id, algo_para)
executor.train_policy(train_timestep)
executor.test_model(test_timestep)

max_stars_repo_name: GuRongfei/BPG-DBS

max_issues_repo_name: GuRongfei/BPG-DBS

max_forks_repo_name: GuRongfei/BPG-DBS


text:
import os
import re
import cv2
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

base_path = os.getcwd()
data = pd.read_csv(base_path+'/10_round_results.csv')
image = cv2.imread((base_path+'/dataset/trainset/faces/face00001.png'),cv2.IMREAD_GRAYSCALE)
feature_names = pd.read_csv(base_path+'/features_names.csv')
for k in range(len((data.J_values)[:])):
    image_with_feature = image.copy()
    string_temp = (feature_names.iloc[(data.J_values)[k]])[0]
    [w,h,i,j] = (re.findall('\d+', string_temp))
    [w,h,i,j] = list(map(int, [w,h,i,j]))

    if ((data.J_values).iloc[k] <= 7440):
     #type2
        a = 255*np.array([1]*h*w + [0]*h*w)
        a.resize(h,2*w)
        image_with_feature[i:i+h,j:j+w*2] = a
    elif ( (data.J_values).iloc[k] <= 14880 ):
    #type 1
        a = 255*np.array([1]*h*w + [0]*h*w)
        a.resize(2*h,w)
        image_with_feature[i:i+h*2,j:w+j] = a
    elif ( (data.J_values).iloc[k] <= 18352 ):
    #type 3
        a = 255*np.array([0]*h*w + [1]*2*h*w + [0]*h*w)
        a.resize(4*h,w)
        image_with_feature[i:i+h*4,j:j+w] = a
    elif ( (data.J_values).iloc[k] <= 21824 ):
    #type 4
        a = 255*np.array([0]*h*w + [1]*2*h*w + [0]*h*w)
        a.resize(h,4*w)
        image_with_feature[i:i+h,j:4*w+j] = a
    else:
    #type 5
        a = 255*np.array([1]*h*w + [0]*h*w + [0]*h*w + [1]*h*w)
        a.resize(2*h,2*w)
        image_with_feature[i:i+2*h,j:2*w+j] = a

    fig = plt.imshow(image_with_feature,cmap=(plt.cm).gray)
    plt.show(fig)


text:
#!/usr/bin/env python3

import numpy as np
from gym.envs.mujoco.walker2d import Walker2dEnv
# import wandb

class Walker2dVelEnv(Walker2dEnv):
    def __init__(self, n_tasks=6, goal: float = 10): #, noise: float = 0):
        self.goal = goal
        # self.noise = noise
        self.max_steps = 1000
        self.step_num = 0  # how many steps passed since environment reset
        self.tasks = np.array([-1.5, -1. , -0.5,  0.5,  1. ,  1.5])
        super(Walker2dVelEnv, self).__init__()

    @property
    def goal(self):
        return self._goal

    @goal.setter
    def goal(self, goal):
        self._goal = goal
        # print(f"Walker2dVel-v0 target speed: {self._goal}")

    # @property
    # def noise(self):
    #     return self._noise

    # @noise.setter
    # def noise(self, noise):
    #     self._noise = noise
    #     print(f"Walker2dVel-v0 action noise std: {self._noise}")

    def step(self, action: np.ndarray):
        self.step_num += 1

        xposbefore = self.sim.data.qpos[0]
        # if self.noise:
        #     action = action + np.random.normal(0, self.noise, size=action.shape)
        self.do_simulation(action, self.frame_skip)
        xposafter, height, ang = self.sim.data.qpos[0:3]

        alive_bonus = 1.0

        forward_vel = (xposafter - xposbefore) / self.dt
        # print("vel:", forward_vel)
        # forward_reward = forward_vel
        ctrl_cost = 1e-3 * np.sum(np.square(action))

        observation = self._get_obs()

        rewards = []
        for task in self.tasks:
            goal_vel = task
            forward_reward = 1.0 * (abs(goal_vel) - abs(forward_vel - goal_vel))
            reward = forward_reward - ctrl_cost + alive_bonus
            rewards.append(reward)
        # done = not (height > 0.8 and height < 2.0 and ang > -1.0 and ang < 1.0)
        done = not (height > 0.5 and height < 2.5 and ang > -1.3 and ang < 1.3) and (self.step_num >= self.max_steps)
        infos = dict(
            reward_forward=forward_reward, reward_ctrl=-ctrl_cost, goal_vel=self.goal, forward_vel=forward_vel, rewards=rewards
        )
        # wandb.log(infos, commit=False)
        return (observation, reward, done, infos)

    def reset(self):
        self.step_num = 0
        return super().reset()

max_stars_repo_name: ArashVahabpour/sog-gail

max_issues_repo_name: ArashVahabpour/sog-gail

max_forks_repo_name: ArashVahabpour/sog-gail