text:
"""
Description: Cleans the crsp database to only include entries with valid returns and
so that every stock is uniquely identified by Permco-datadate.

I rename some variables, and collapse share classes. In the combined share classes, I
add up the market cap of each share class to get to the total market cap of that
company at that time. Returns are also computed as market cap weighted returns of the
individual share classes.
"""

# Custom functions for data cleaning
from utils import *
import numpy as np
from copy import copy

# Proper data cleaning
################ Import Data ################
print_message('Loading Data')
crsp_raw = pd.read_csv('../Data/crsp.txt', sep = '\t', low_memory = False)#, nrows = 10000)

# Filter down to only common shares
crsp_raw = crsp_raw.loc[(crsp_raw['SHRCD'] == 10) | (crsp_raw['SHRCD'] == 11)]

print('Data Size:')
print(crsp_raw.shape)
print(str(crsp_raw.shape[0] * crsp_raw.shape[1]) + ' observations')

################ Setting Types ################
print_message('Setting Types')
crsp_datatypes = {'date_vars': ['date'],
                 'float_vars': ['BIDLO', 'ASKHI', 'PRC', 'RET', 'DLRET', 'BID', 'ASK', 'RETX', 'CFACPR', 'CFACSHR'],
                 'int_vars': ['PERMNO', 'PERMCO', 'HSICCD', 'CUSIP', 'SHROUT', 'VOL', 'EXCHCD']}
crsp = clean_data(copy(crsp_raw), crsp_datatypes)

################ Setting Types ################
print_message('Renaming Variables')
crsp_names = {'RET': 'Return',
              'SHROUT': 'Shares Outstanding on Trading Day',
              'COMNAM': 'Company Name',
              'EXCHCD': 'Exchange Code',
              'TICKER': 'Ticker',
              'date': 'datadate',
              'PERMNO': 'Permno',
              'PERMCO': 'Permco',
              'PRC': 'Price with Flag',
              'BID': 'Bid',
              'ASK': 'Ask',
              'VOL': 'Volume on Trading Day',
              'SHRCLS': 'Share Class',
              'CFACPR': 'Price Adjustment Factor',
              'CFACSHR': 'Share Adjustment Factor'}

crsp = crsp.rename(index = str, columns = crsp_names)
crsp = crsp[list(crsp_names.values())]

# Make a few fundamental variables
print_message('Create new features')
crsp['Imputed Price'] = (crsp['Price with Flag'] < 0)
crsp['Price on Trading Day'] = np.abs(crsp['Price with Flag'])
crsp['Price'] = crsp['Price on Trading Day'] / crsp['Price Adjustment Factor']
crsp['Shares Outstanding'] = crsp['Shares Outstanding on Trading Day'] * crsp['Share Adjustment Factor']
crsp['Volume'] = crsp['Volume on Trading Day'] * crsp['Share Adjustment Factor']
crsp['Market Cap (Billions, CRSP)'] = crsp['Shares Outstanding'] * crsp['Price'] / 1e6
crsp = crsp.drop(['Price on Trading Day', 'Shares Outstanding on Trading Day', 'Volume on Trading Day'], axis = 1)

################
print_message('Aggregating share classes')
AGG_VAR_TYPES = {'First': ['Company Name', 'Permno', 'Ticker', 'Price', 'Bid', 'Ask', 'Exchange Code'],
                 'Add': ['Market Cap (Billions, CRSP)'],
                 'Market Cap Weighted Sum': ['Return'],
                 'Price Weighted Sum': ['Volume']}

ALL_CRSP_VAR = [item for sublist in AGG_VAR_TYPES.values() for item in sublist]

def agg_share_classes(company_on_date):
    """
    Given a dataframe grouped on Permco and datadate, aggregates up the key variables from the individual
    share classes. THe operation done on each variable is defined by the global dictionary AGG_

    :param company_on_date - a dataframe from a pandas groupby object with all the variables inside ALL_CRSP_VAR
    :return a dataframe with one row that has all the variables in ALL_CRSP_VAR, agrgregated across the share classes
    """

    company_on_date.safe_drop(['_Group'], inplace = True) # Addition to make it work well with the parallel process

    # If only one share class
    if company_on_date.shape[1] <= 1:
        return company_on_date[ALL_CRSP_VAR]

    # Then for the remainign cases
    company_on_date['Market Cap (Billions, CRSP)'].fillna(0)
    company_on_date.sort_values(by = ['Market Cap (Billions, CRSP)'], ascending = False, inplace = True)
    new_frame = company_on_date.iloc[0, :][AGG_VAR_TYPES['First']]

    for v in AGG_VAR_TYPES['Add']:
        new_frame[v] = company_on_date[v].fillna(0).sum()

    for v in AGG_VAR_TYPES['Market Cap Weighted Sum']:
        valid = ~company_on_date[v].isnull()
        denominator = company_on_date.loc[valid, 'Market Cap (Billions, CRSP)'].sum()

        if denominator > 0:
            new_frame[v] = (company_on_date.loc[valid, v] * company_on_date.loc[
                valid, 'Market Cap (Billions, CRSP)']).sum() / denominator
        else:
            new_frame[v] = company_on_date[v][0]

    for v in AGG_VAR_TYPES['Price Weighted Sum']:
        valid = ~company_on_date[v].isnull()
        denominator = new_frame['Price']

        if denominator > 0:
            new_frame[v] = (company_on_date.loc[valid, v] * company_on_date.loc[
                valid, 'Price']).sum() / denominator
        else:
            new_frame[v] = company_on_date[v][0]

    return new_frame

# First, pull every permco-date that isn't paired
crsp = crsp.set_index(['Permco', 'datadate'])
crsp_counts = check_unique(crsp, ['Permco', 'datadate'])

# Then isolate the problem observations
crsp_merge = crsp_counts.join(crsp, how = 'outer')
problem_children = crsp_merge.loc[~pd.isnull(crsp_merge['Count']), ALL_CRSP_VAR]

if problem_children.shape[0] > 0:
    good_children = crsp_merge.loc[pd.isnull(crsp_merge['Count']), ALL_CRSP_VAR]
    merged_problems = parallel_apply(problem_children, ['Permco', 'datadate'], agg_share_classes, 2, None)
    merged_problems = merged_problems[ALL_CRSP_VAR]

    # Combine the dataframes together again
    crsp_merge = pd.concat([good_children[ALL_CRSP_VAR], merged_problems])

print_message('Dropping Null Returns')
crsp_merge = crsp_merge.loc[~pd.isnull(crsp_merge['Return'])]

#######
print_message('Verifying data integrity')
crsp_merge = crsp_merge.safe_index(['Permco', 'datadate'])

# Check final assumptions on the data
# 1. Permco + datadate uniquely identify each observation
# 2. For each permco, there is a continuous stream of returns without interruption

# Unique identification
duplicated_values = check_unique(crsp_merge, ['Permco', 'datadate'])
print('Duplicated permco-date pairs')
print(duplicated_values)
assert(duplicated_values.shape[0] == 0)

# Continuity of returns
valid_returns = parallel_apply(crsp_merge, ['Permco'], continuous_index, 2, None)
discontinuous_returns = valid_returns.loc[~valid_returns]
print('Companies without continuous returns')
print(discontinuous_returns)
assert(discontinuous_returns.shape[0] == 0)

################
print_message('Outputting Data')
crsp_merge.to_hdf('../Output/crsp.h5', 'crsp')

with open('../Logs/crsp.log', 'w') as f:
    f.write('Raw CRSP file has been written')


text:
#!/usr/bin/env python
# coding: utf-8

# In[ ]:

import sys
sys.path.insert(0,"./../") #so we can import our modules properly

# In[ ]:

# iPhython
from IPython.core.display import display, HTML
display(HTML("<style>.container { width:90% !important; }</style>"))

from matplotlib import rcParams
import matplotlib.pyplot as plt
get_ipython().run_line_magic('matplotlib', 'notebook')

import numpy as np
import pandas as pd

# # House E

# In[ ]:

path_to_house = "./datasets/dfE_300s.hdf"
df_house = pd.read_hdf(path_to_house)
print('\n\nStart time: {}'.format(df_house.index[1]))
print('End time: {}'.format(df_house.index[-1]))
print(df_house.columns)

# ## Total Imported Power and Total Submetered Power

# In[ ]:

meters = ['E_dehumidifier_power', 'E_dishwasher_power', 'E_gasheating_pump_power', 'E_solarheating_pump_power', 'E_tumble_dryer_power', 'E_washing_machine_power']

# Investigate only points in time where all values are available
cols = ['E_imp_power']
cols.extend(meters)
df_house_noNAN = df_house.loc[:,cols].dropna(axis=0, how='any')

# Check if total consumed power is larger than total of appliances
delta = df_house_noNAN.loc[:,meters].sum(axis=1) - df_house_noNAN.loc[:,'E_imp_power']
tmt = delta > 0

if np.any(tmt):
    # investigate problematic cases
    print("Found {} out of {} ({:.2}%) values to be problematic.".format(tmt.sum(), len(tmt), tmt.sum()/len(tmt)*100))
    print("")
    print("Statistics of problematic values:")
    print(delta[tmt].describe())
    print()
    print((delta[tmt]/df_house_noNAN.loc[tmt, 'E_imp_power']).describe())
else:
    print("Total of all appliances is smaller than total consumed energy for all measurement points.")

# In[ ]:

l = ['Total Appliances', 'Total Consumed']
l.extend(meters)
if np.any(tmt):
    ## plot params ##
    ha = 6
    ncols = 8

    nrows = np.sum(tmt)//ncols+1
    nrows = 4
    fig, ax = plt.subplots(figsize=(17,3*nrows), ncols=ncols, nrows=nrows)
    idxs = np.nonzero(tmt)[0]
    for i, idx in enumerate(idxs[0:32]):
        minidx = idx-ha
        maxidx = idx+ha
        df_house_noNAN.loc[:,meters].sum(axis=1).iloc[minidx:maxidx].plot(ax=ax[i//ncols, i%ncols], label='Total Appliances')
        df_house_noNAN.iloc[minidx:maxidx].loc[:,'E_imp_power'].plot(ax=ax[i//ncols, i%ncols], label='Total Consumed')
        for meter in meters:
            df_house_noNAN.iloc[minidx:maxidx].loc[:,meter].plot(ax=ax[i//ncols, i%ncols], label=meter)
        ax[i//ncols, i%ncols]
    fig.legend(l, loc='upper center')
else:
    print("Total of all appliances is smaller than total consumed energy for all measurement points.")

# # House E - Weather

# In[ ]:

path_to_house = "./datasets/dfE_3600s.hdf"
df_weather = pd.read_hdf(path_to_house)
print('\n\nStart time: {}'.format(df_weather.index[1]))
print('End time: {}'.format(df_weather.index[-1]))
print(df_weather.columns)

# In[ ]:

ext: py

max_stars_repo_name: brauliobarahona/RAPT-dataset

max_issues_repo_name: brauliobarahona/RAPT-dataset

max_forks_repo_name: brauliobarahona/RAPT-dataset

reason: import numpy


text:
#!/usr/bin/env python3

import os
import psycopg2
import pandas as pd
import numpy as np
import magonote_funcs as mf
from db_info import db_name, db_user, db_pw, db_host

"""

pull out comment scores, write a tsv file

"""
db_connection = psycopg2.connect( database=db_name, user=db_user, host=db_host, password=db_pw )
cur = db_connection.cursor()

# get userids, put in dict
# key is slug, val is userid
# just you know, to make it inconvenient
user_dict = dict()
cur.execute("SELECT user_id, slug FROM itch_user")
for user_row in cur:
    user_dict[user_row[1]] = user_row[0]

test_query = """
    SELECT user_slug, game_id, SUM(div_len) AS score
    FROM itch_post_div_lengths
    GROUP BY user_slug, game_id
    ORDER BY RANDOM()
    LIMIT 5000
"""

query = """
    SELECT user_slug, game_id, SUM(div_len) AS score
    FROM itch_post_div_lengths
    GROUP BY user_slug, game_id
"""

file_list = list()
cur.execute(query)
for post_row in cur:
    if post_row[0] in user_dict:
        user_id = user_dict[post_row[0]]
        game_id = post_row[1]
        rating = post_row[2]
        stamp = 123456
        file_row = "{}  {}  {}  {}\n".format(user_id, game_id, rating, stamp)
        file_list.append(file_row)

with open ('test_post_data.idf', 'w') as fh:
    for file_line in file_list:
        fh.write(file_line)

max_stars_repo_name: pagrus/galvanize-magonote

max_issues_repo_name: pagrus/galvanize-magonote

max_forks_repo_name: pagrus/galvanize-magonote


text:
# Run this cell (Shift+Enter) to show application here
import JupyterNotebooksLib as slicernb
slicernb.AppWindow(contents='full')

#download file and load volume rendering to the scene
slicer.mrmlScene.Clear()

volume = slicernb.downloadFromURL(
    uris="https://github.com/allihuwa/Content-Production-for-Improving-Cancer-Treatment-Worklflow/raw/master/pelvisvolume.nrrd",
    fileNames="pelvisvolume.nrrd",
    nodeNames="pelvisvolume")[0]

slicernb.showVolumeRendering(volume)

#Enable segmentation and add sphere to the segmentation node

segmentationNode = slicer.vtkMRMLSegmentationNode()
slicer.mrmlScene.AddNode(segmentationNode)
segmentationNode.CreateDefaultDisplayNodes() # only needed for display
segmentationNode.SetReferenceImageGeometryParameterFromVolumeNode(volume)

#Sphere 1
tumorSeed = vtk.vtkSphereSource()
tumorSeed.SetCenter(0, -50, 3)
tumorSeed.SetRadius(10)
tumorSeed.Update()
segmentationNode.AddSegmentFromClosedSurfaceRepresentation(tumorSeed.GetOutput(), "Tumor", [1.0,0.0,0.0])

# Compute histogram of intensities corresponding to sphere location
#with respect to the volume rendering location of intersection
################################################

labelValue = 1  # label value of first segment

# Get segmentation as labelmap volume node
labelmapVolumeNode = slicer.mrmlScene.AddNewNodeByClass('vtkMRMLLabelMapVolumeNode')
slicer.modules.segmentations.logic().ExportVisibleSegmentsToLabelmapNode(segmentationNode, labelmapVolumeNode, volume)

# Extract all voxels of the segment as numpy array
volumeArray = slicer.util.arrayFromVolume(volume)
labelArray = slicer.util.arrayFromVolume(labelmapVolumeNode)
segmentVoxels = volumeArray[labelArray==labelValue]

# Compute histogram
import numpy as np
histogram = np.histogram(segmentVoxels, bins=50)

# Plot histogram
################################################

slicer.util.plot(histogram, xColumnIndex = 1)


text:
import os
import re
import time
import copy
import scipy
import socket
import struct
from typing import List, Tuple
import numpy as np
import crcmod.predefined
from scipy import integrate
import matplotlib.pyplot as plt
from bitstring import ConstBitStream

def createRandomData(filepath: str, datasize: int) -> None:
    """ Create random data of size DATASIZE and store it in FILEPATH """
    np.random.seed()
    file = open(filepath, "w")
    data = np.random.randint(2, size=datasize).tolist()
    # print(data)
    for i in range(datasize):
        file.write(str(data[i]))

def dec2arr(dec_value: int, length: int) -> np.ndarray:
    """ Receive a decimal DEC_VALUE and transform it to a binary ndarray with size of LENGTH """
    bin_str = bin(dec_value)[2:]
    # print(bin_str)
    bin_array = np.zeros(length, dtype=np.int)
    bin_array[length-len(bin_str):length] = np.array(list(bin_str))
    return bin_array

def arr2dec(bin_arr: np.ndarray) -> int:
    """ Receive a binary ndarray BIN_ARR and transform to a decimal value """
    dec = 0
    factor = 1
    for i in bin_arr[::-1]:
        dec += i * factor
        factor *= 2
    return dec

def arr2str(bin_arr: np.ndarray) -> str:
    """ Receive a binary ndarray BIN_ARR and transform it to a binary string """
    string = ''.join([str(x) for x in bin_arr])
    return string

def ip2arr(ip_str: str) -> np.ndarray:
    """ Receive a string of IP address and transform it to a binary ndarray """
    result = np.array([])
    for seg in ip_str.split('.'):
        result = np.concatenate((result, dec2arr(int(seg), 8)), axis=0)
    return result

def generateSplitBinData(raw_data) -> List[np.ndarray]:
    """ Receive a string or a ndarray of RAW_DATA, split it according to a fixed size """
    bin_data_list = []
    raw_copy = copy.deepcopy(raw_data)

    if isinstance(raw_data, str):
        str_data_list = []
        while len(raw_copy) > 40:
            str_data_list.append(raw_copy[:40])
            raw_copy = raw_copy[40:]
        else:
            str_data_list.append(raw_copy)

        for line in str_data_list:
            binary_line = np.array([])
            for i in line:
                binary = dec2arr(ord(i), 8)
                binary_line = np.concatenate((binary_line, binary), axis=0)
            bin_data_list.append(binary_line.astype(np.uint8))
    elif isinstance(raw_data, np.ndarray):
        raw_copy = raw_copy.reshape((-1, ))
        while raw_copy.size > 40:
            bin_data_list.append(raw_copy[:40])
            raw_copy = raw_copy[40:]
        else:
            bin_data_list.append(raw_copy)
    else:
        raise AssertionError("Unsupported data type to split!")

    return bin_data_list

def generateCRC(bin_arr: np.ndarray, mode: str = 'crc-16') -> str:
    """ Calculate crc with given MODE of an encoded 16-based binary ndarray BIN_ARR """
    crc_func = crcmod.predefined.Crc(mode)
    hexData = hex(int(arr2str(bin_arr), 2))[2:]
    crc_func.update(hexData.encode())
    crc_result = hex(crc_func.crcValue)
    return crc_result

def generateCarrierWave(fs: int, fc: int) -> np.ndarray:
    """ Generate carrier wave with given FS and FC """
    one_second = np.linspace(0, 1, fs)
    carrier = np.sin(2 * np.pi * fc * one_second)
    return carrier

def generatePreambleWave(fs: int, wave_len_per_bit: int) -> np.ndarray:
    """ Generate carrier wave with given FS and WAVE_LEN_PER_BIT """
    one_second = np.linspace(0, 1, fs)
    preamble = np.append(np.linspace(300, 2100, int(wave_len_per_bit / 2 * 128)), np.linspace(2100, 300, int(wave_len_per_bit / 2 * 128)))
    # There's small difference between scipy's cumtrapz and Matlab's cumtrapz
    omega = 2 * np.pi * scipy.integrate.cumtrapz(preamble, one_second[:int(wave_len_per_bit * 128)], initial=0)
    preamble_wave = np.sin(omega)
    # plt.plot(one_second[:440], preamble_wave)
    # plt.show()
    return preamble_wave

def smooth(arr: np.ndarray, span: int) -> np.ndarray:
    """ Smooth the ndarray ARR with the given window size SPAN """
    out0 = np.convolve(arr, np.ones(span, dtype=int), 'valid') / span
    r = np.arange(1, span - 1, 2)
    start = np.cumsum(arr[:span - 1])[::2] / r
    stop = (np.cumsum(arr[:-span:-1])[::2] / r)[::-1]
    return np.concatenate((start, out0, stop))

def do_checksum(source_string: bytes) -> int:
    """ Verify the packet integrity with SOURCE_STRING """
    sum = 0
    max_count = (len(source_string) / 2) * 2
    count = 0
    while count < max_count:
        val = source_string[count + 1] * 256 + source_string[count]
        sum = sum + val
        sum = sum & 0xffffffff
        count = count + 2

    if max_count < len(source_string):
        sum = sum + ord(source_string[len(source_string) - 1])
        sum = sum & 0xffffffff

    sum = (sum >> 16) + (sum & 0xffff)
    sum = sum + (sum >> 16)
    answer = ~sum
    answer = answer & 0xffff
    answer = answer >> 8 | (answer << 8 & 0xff00)
    return answer

def generateICMP(ICMP_ECHO: int, target_host: str, ID: int) -> bytes:
    """ Gnenrate ICMP packet with ICMP_ECHO type, TARGET_HOST, ID """
    target_addr = socket.gethostbyname(target_host)
    my_checksum = 0
    # Create a dummy heder with a 0 checksum.
    header = struct.pack("bbHHh", ICMP_ECHO, 0, my_checksum, ID, 1)
    bytes_In_double = struct.calcsize("d")
    data = (192 - bytes_In_double) * "Q"
    data = struct.pack("d", time.time()) + bytes(data.encode('utf-8'))
    # Get the checksum on the data and the dummy header.
    my_checksum = do_checksum(header + data)
    header = struct.pack(
        "bbHHh", ICMP_ECHO, 0, socket.htons(my_checksum), ID, 1
    )
    packet = header + data
    return packet

def arrDiff(a: np.ndarray, b: np.ndarray) -> None:
    """ Output the correct rate between two ndarrays A and B """
    correct_num = np.sum(a == b)
    print("Correct Rate: ", correct_num / a.size * 100, "%", sep='')

def fileDiff(a: np.ndarray, b: np.ndarray) -> None:
    """ Output the correct rate between two binary files A and B """
    print("Diffing...")
    input_size, output_size = os.path.getsize(a) * 8, os.path.getsize(b) * 8
    if output_size <= 0:
        print(">> OUTPUT.bin is EMPTY, Accuracy: 0%")
        return
    elif output_size < input_size:
        print(">> Partial Data Received, received: ", output_size)

    compare_size = min(input_size, output_size)
    input_stream, output_stream = ConstBitStream(filename=a), ConstBitStream(filename=b)
    input_bits, output_bits = input_stream.read(compare_size), output_stream.read(compare_size)
    input_arr, output_arr = np.array(list(input_bits), dtype=np.uint8), np.array(list(output_bits), dtype=np.uint8)

    correct_num = np.sum(input_arr == output_arr)
    print(">> Received Data Accuracy: ", correct_num / input_arr.size * 100, "%", sep='')

def decodeBinInfoData(bin_data: List[str]) -> List[str]:
    """ Decode information data from BIN_DATA """
    decoded_data = []
    for line in bin_data:
        temp_line = ""
        for bin_asc in re.findall(r'.{8}', line):
            dec_asc = int(bin_asc, 2)
            ch = chr(dec_asc)
            temp_line += ch
        decoded_data.append(temp_line)
    return decoded_data

def decodeBinFileData(bin_data: List[str]) -> str:
    """ Decode file data from BIN_DATA """
    return "".join(bin_data)

def getCmd() -> Tuple[str]:
    """ Parse user's commands """
    while True:
        cmdline = input("ftp> ")
        cmdline = cmdline.strip()
        cmd_stack = cmdline.split()
        if len(cmd_stack) == 0:
            continue

        cmd = cmd_stack[0].upper()

        if cmd == "USER":
            if len(cmd_stack) > 1:
                user = cmd_stack[1]
            else:
                user = "anonymous"
            return cmd, user

        elif cmd == "PASS":
            if len(cmd_stack) > 1:
                passwd = cmd_stack[1]
            else:
                passwd = ""
            return cmd, passwd

        elif cmd == "PWD":
            return (cmd, )

        elif cmd == "CWD":
            if len(cmd_stack) > 1:
                dir = cmd_stack[1]
                return cmd, dir
            else:
                print("You should provide the remote-directory")

        elif cmd == "PASV":
            return (cmd, )

        elif cmd == "LIST":
            if len(cmd_stack) > 1:
                dir = cmd_stack[1]
            else:
                dir = ""
            return cmd, dir

        elif cmd == "RETR":
            if len(cmd_stack) == 2:
                remote_file = cmd_stack[1]
                return cmd, remote_file
            elif len(cmd_stack) == 3:
                remote_file = cmd_stack[1]
                local_file = cmd_stack[2]
                return cmd, remote_file + " " + local_file
            else:
                print("You should provide the remote-file (and maybe local-file)")

        elif cmd == "QUIT":
            return (cmd, )

        else:
            print("Wrong Command! Try Again!")

max_stars_repo_path: Project 4/utils.py

max_stars_repo_name: MercurialJD/Athernet

max_issues_repo_path: Project 4/utils.py

max_issues_repo_name: MercurialJD/Athernet

max_forks_repo_path: Project 4/utils.py

max_forks_repo_name: MercurialJD/Athernet

reason: import numpy,import scipy,from scipy


text:
import pymongo
client = pymongo.MongoClient("mongodb://mdbadmin:fwYHdDtTym6lNmyoiJPTfZzTW0jiokx5@aimdb.devhcm.local:30000/admin")
my_db = client['chatbot_quangminhtien']
my_col = my_db['conversation']
# document = my_col.find({}, {"_id":0})
# conversation = []
# for item in document:
#     conversation.append(item)
# print(conversation)
# libraries
import random
import numpy as np
import pickle
import json
from flask import Flask, render_template, request
from flask_ngrok import run_with_ngrok
import nltk
from keras.models import load_model
from nltk.stem import WordNetLemmatizer
lemmatizer = WordNetLemmatizer()
from backend.process.product_handler import check_condition, reset_conversation
# chat initialization
model = load_model("chatbot_model.h5")
# intents = json.loads(open("intents.json").read())
with open('intents.json', encoding='utf-8') as fh:
    intents = json.load(fh)
words = pickle.load(open("words.pkl", "rb"))
classes = pickle.load(open("classes.pkl", "rb"))

app = Flask(__name__)
run_with_ngrok(app)

@app.route("/")
def home():
    return render_template("index.html")

@app.route("/reset", methods=["GET"])
def reset():
    return reset_conversation()

@app.route("/get", methods=["POST"])
def chatbot_response():
    document = my_col.find({}, {"_id":0})
    conversation_info = []
    for item in document:
        conversation_info.append(item)
    lst_entity = conversation_info[-1]['information']

    msg = request.form["msg"]
    ints = predict_class(msg, model)
    res = getResponse(ints, intents)
    tag = ints[0]['intent']
    if float(ints[0]['probability']) < 0.99 or not res:
        document = my_col.find({}, {"_id":0})
        conversation = []
        for item in document:
            conversation.append(item)
        pre_tag = conversation[-1]['pre_tag']
        if float(ints[0]['probability']) < 0.99:
            res, tag, lst_entity = check_condition(pre_tag, intents, msg, lst_entity)
        else:
            res, tag, lst_entity = check_condition(tag, intents, msg, lst_entity)

    # for i in range(len(lst_entity)):
    #     res = res.replace('{', '')
    #     res = res.replace('}', '')
    #     res = res.replace(list(lst_entity.keys())[i], list(lst_entity.values())[i])

    query = {"sender_id": "1"}
    my_col.update_one(query, {"$set": {
        "sender_id": "1",
        "information": lst_entity,
        "pre_tag": tag
    }})
    return res

# chat functionalities
def clean_up_sentence(sentence):
    sentence_words = nltk.word_tokenize(sentence)
    sentence_words = [lemmatizer.lemmatize(word.lower()) for word in sentence_words]
    return sentence_words

# return bag of words array: 0 or 1 for each word in the bag that exists in the sentence
def bow(sentence, words, show_details=True):
    # tokenize the pattern
    sentence_words = clean_up_sentence(sentence)
    # bag of words - matrix of N words, vocabulary matrix
    bag = [0] * len(words)
    for s in sentence_words:
        for i, w in enumerate(words):
            if w == s:
                # assign 1 if current word is in the vocabulary position
                bag[i] = 1
                if show_details:
                    print("found in bag: %s" % w)
    return np.array(bag)

def predict_class(sentence, model):
    # filter out predictions below a threshold
    p = bow(sentence, words, show_details=False)
    res = model.predict(np.array([p]))[0]
    ERROR_THRESHOLD = 0.25
    results = [[i, r] for i, r in enumerate(res) if r > ERROR_THRESHOLD]
    # sort by strength of probability
    results.sort(key=lambda x: x[1], reverse=True)
    return_list = []
    for r in results:
        return_list.append({"intent": classes[r[0]], "probability": str(r[1])})
    return return_list

def getResponse(ints, intents_json):
    tag = ints[0]["intent"]
    list_of_intents = intents_json["intents"]
    for i in list_of_intents:
        if i["tag"] == tag:
            if i["responses"]:
                result = random.choice(i["responses"])
                break
            else:
                result = ""
    return result

if __name__ == "__main__":
    app.run()


text:
import numpy as np
import pandas as pd

dataset = pd.read_csv('../Dataset/Crimes_2001_to_2022 _v1.csv', low_memory=False)
print('Columns in dataset: ', dataset.columns)

X = dataset.iloc[:, :-1].values
y = dataset.iloc[:, 1].values

print('\n Displaying DataSets for X \n', X)
print('\n Displaying DataSets for y \n', y)

print('Displaying dataset,', dataset.head(5))

from sklearn.utils import shuffle

dataset = shuffle(dataset)
print("After shuffling the dataset")

print('Displaying dataset,', dataset.head(5))

from sklearn.impute import SimpleImputer

imputer = SimpleImputer(missing_values=np.nan, strategy='most_frequent')
imputer = imputer.fit(X[:, :22])

print(' Splitting the Values In X \n')
print(' Displaying DataSets for X \n', X)

X[:, :22] = imputer.transform(X[:, :22])
print('Filling the value using Most Frequent method: \n')
print(' Displaying DataSets for X \n', X)

from sklearn.preprocessing import LabelEncoder, OneHotEncoder

# Thus, we should also use OneHotEncoding by adding dummy columns as per number of distinct values in column country
onehotencoder = OneHotEncoder()
X = onehotencoder.fit_transform(X).toarray()
labelEncoder_y = LabelEncoder()
y = labelEncoder_y.fit_transform(y)
print('\n DataSets After Encoding and Sorting the column \n')
print('Displaying DataSets for X \n', X)
print('\n Displaying DataSets for y \n', y)

from sklearn.preprocessing import LabelEncoder, OneHotEncoder

labelEncoder_X = LabelEncoder()
X[:, 22] = labelEncoder_X.fit_transform(X[:, 22])
print('\n Displaying DataSets in X after Encoding the first column \n', X)

# Splitting the dataset into the Training set and Test set
from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)

from sklearn.linear_model import LogisticRegression

lr = LogisticRegression(solver='liblinear', multi_class='ovr')
lr.fit(X_train, y_train)
print('Accuracy of Logistic Regression', lr.score(X_test, y_test))

from sklearn.svm import SVC

svm = SVC(gamma='auto')
svm.fit(X_train, y_train)
print('Accuracy of SVM', svm.score(X_test, y_test))

from sklearn.neighbors import KNeighborsClassifier

knn = KNeighborsClassifier(n_neighbors=10)
knn.fit(X_train, y_train)
print('Accuracy of KNN', knn.score(X_test, y_test))

from sklearn.naive_bayes import GaussianNB

nb = GaussianNB()
nb.fit(X_train, y_train)
print('Accuracy of Naive Bayes', nb.score(X_test, y_test))

from sklearn.model_selection import StratifiedKFold

folds = StratifiedKFold(n_splits=2, shuffle=True, random_state=0)
scores_logistic = []
for train_index, test_index in folds.split(X_train, y_train):
    X_train_fold, X_test_fold = X[train_index], X[test_index]
    y_train_fold, y_test_fold = y[train_index], y[test_index]
    lr = LogisticRegression(solver='liblinear', multi_class='ovr')
    lr.fit(X_train_fold, y_train_fold)
    print('Accuracy of Logistic Regression by k fold: ', scores_logistic.append(lr.score(X_train_fold, y_train_fold)))


text:
#!/usr/bin/env python
# coding: utf-8

# In[1]:

import matplotlib.pyplot as plt

import pandas as pd
import numpy as np
import statistics
import math
from sklearn.model_selection import train_test_split
import random
import sklearn
from sklearn import ensemble
from itertools import chain
from typing import TextIO
import re

from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import GroupKFold
from sklearn.model_selection import PredefinedSplit

from sklearn import linear_model
from sklearn.linear_model import Ridge
from sklearn.linear_model import ElasticNet
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.ensemble import RandomForestClassifier
from sklearn import svm
from sklearn.svm import SVC
from sklearn.svm import LinearSVC
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import GradientBoostingClassifier
#from sklearn.ensemble import VotingRegressor
from sklearn.metrics import mean_squared_error
from sklearn.metrics import roc_auc_score, average_precision_score, roc_curve, precision_recall_curve, confusion_matrix

# In[2]:

# data
genes = ['RPS14', 'CDC5L', 'POLR2I', 'RPS7', 'XAB2', 'RPS19BP1', 'RPL23A', 'SUPT6H', 'PRPF31', 'U2AF1', 'PSMD7',
         'Hsp10', 'RPS13', 'PHB', 'RPS9', 'EIF5B', 'RPS6', 'RPS11', 'SUPT5H', 'SNRPD2', 'RPL37', 'RPSA', 'COPS6',
         'DDX51', 'EIF4A3', 'KARS', 'RPL5', 'RPL32', 'SF3A1', 'RPS3A', 'SF3B3', 'POLR2D', 'RPS15A', 'RPL31', 'PRPF19',
         'SF3B2', 'RPS4X', 'CSE1L', 'RPL6', 'COPZ1', 'PSMB2', 'RPL7', 'PHB2', 'ARCN1', 'RPA2', 'NUP98', 'RPS3', 'EEF2',
         'USP39', 'PSMD1', 'NUP93', 'AQR', 'RPL34', 'PSMA1', 'RPS27A']

genes_filter_1 = ['RPS6', 'PRPF19', 'RPL34', 'Hsp10', 'POLR2I', 'EIF5B', 'RPL31',
       'RPS3A', 'CSE1L', 'XAB2', 'PSMD7', 'SUPT6H', 'EEF2', 'RPS11',
       'SNRPD2', 'RPL37', 'SF3B3', 'DDX51', 'RPL7', 'RPS9', 'KARS',
       'SF3A1', 'RPL32', 'PSMB2', 'RPS7', 'EIF4A3', 'U2AF1', 'PSMA1',
       'PHB', 'POLR2D', 'RPSA', 'RPL23A', 'NUP93', 'AQR', 'RPA2',
       'SUPT5H', 'RPL6', 'RPS13', 'SF3B2', 'RPS27A', 'PRPF31', 'COPZ1',
       'RPS4X', 'PSMD1', 'RPS14', 'NUP98', 'USP39', 'CDC5L', 'RPL5',
       'PHB2', 'RPS15A', 'RPS3', 'ARCN1', 'COPS6']

gene_split_index = {}
for i in range(len(genes_filter_1)):
    gene = genes_filter_1[i]
    gene_split_index[gene]= math.floor(i/6)

base_positions = {
    'A': 0,
    'T': 1,
    'C': 2,
    'G': 3,
    0: 'A',
    1: 'T',
    2: 'C',
    3: 'G',
}

def create_gene_splits_filter1_kfold(gene_strings, values_to_split: list, kfold, split):
    # use number [0, 1, 2, 3, 4,...] as index
    genes_filter_1 = ['RPS6', 'PRPF19', 'RPL34', 'Hsp10', 'POLR2I', 'EIF5B', 'RPL31',
       'RPS3A', 'CSE1L', 'XAB2', 'PSMD7', 'SUPT6H', 'EEF2', 'RPS11',
       'SNRPD2', 'RPL37', 'SF3B3', 'DDX51', 'RPL7', 'RPS9', 'KARS',
       'SF3A1', 'RPL32', 'PSMB2', 'RPS7', 'EIF4A3', 'U2AF1', 'PSMA1',
       'PHB', 'POLR2D', 'RPSA', 'RPL23A', 'NUP93', 'AQR', 'RPA2',
       'SUPT5H', 'RPL6', 'RPS13', 'SF3B2', 'RPS27A', 'PRPF31', 'COPZ1',
       'RPS4X', 'PSMD1', 'RPS14', 'NUP98', 'USP39', 'CDC5L', 'RPL5',
       'PHB2', 'RPS15A', 'RPS3', 'ARCN1', 'COPS6']
    assert split >= 0 and split < kfold
    if kfold == 9:
        val_genes = genes_filter_1[split * 6: (split + 1) * 6]
        if split != 8:
            test_genes = genes_filter_1[((split + 1) * 6): (split + 2) * 6]
        else:
            test_genes = genes_filter_1[0:6]
    print('val:', val_genes)
    print('test:', test_genes)

    val_ids = list(chain(*[np.where(gene_strings == g)[0] for g in val_genes]))
    test_ids = list(chain(*[np.where(gene_strings == g)[0] for g in test_genes]))
    train_ids = list((set(range(len(gene_strings))) - set(val_ids) - set(test_ids)))

    train = [[arr[i] for i in train_ids] for arr in values_to_split]
    val = [[arr[i] for i in val_ids] for arr in values_to_split]
    test = [[arr[i] for i in test_ids] for arr in values_to_split]

    return train, val, test

def normalize(a: np.ndarray):
    """
    :param a: numpy array of size N x D, where N is number of examples, D is number of features
    :return: a, normalized so that all feature columns are now between 0 and 1
    """
    a_normed, norms = sklearn.preprocessing.normalize(a, norm='max', axis=0, return_norm=True)
    print("Norms:", norms)
    return a_normed

def one_hot_encode_sequence(seq, pad_to_len=-1):
    output_len = len(seq)
    if pad_to_len > 0:
        assert pad_to_len >= output_len
        output_len = pad_to_len

    encoded_seq = np.zeros((output_len, 4), dtype=np.float32)
    for i, base in enumerate(seq):
        encoded_seq[i][base_positions[base]] = 1
    return encoded_seq

dataset_filtered_csv_path = '../../../data/integrated_guide_feature_filtered_f24_mismatch3_all_flanks.csv'

#dataset
dataframe = pd.read_csv(dataset_filtered_csv_path)
dataframe = dataframe[dataframe['gene'].isin(genes_filter_1)] #filter out 1 gene

num_examples = len(dataframe['gene'].values)

encoded_guides = [one_hot_encode_sequence(guide).flatten() for guide in dataframe['guide'].values]
#encoded_linearfold = [one_hot_encode_linearfold(lin_seq_dict[guide], remove_universal_start=True) for guide in
#                      dataframe['guide'].values]

#classification
#classes = dataframe['binary_relative_ratio'].values
classes = dataframe['binary_relative_ratio_075f'].values
#classes = dataframe['top 20 pct per gene'].values
outputs = classes.astype(np.float32)

all_cols = [encoded_guides, outputs]

#tr, val, te = create_gene_splits_kfold(dataframe['gene'].values, all_cols, 11, 4)

# group label to split
groups = dataframe['gene'].values
# predefined split index
for g in gene_split_index.keys():
    dataframe.loc[dataframe['gene']== g,'predefined split index']= gene_split_index[g]
ps = PredefinedSplit(dataframe['predefined split index'].values)
print(ps.get_n_splits())

# hp tuning

#GradientBoostingClassifier
gb = ensemble.GradientBoostingClassifier(random_state=0)
grid = {'n_estimators':[100,200,400,800,1000,1200,1500,1800,2000],'max_depth':[2,4,8],'max_features':['auto','sqrt','log2']}

#gs = GridSearchCV(gb, grid, cv=GroupKFold(n_splits=5))
#gs.fit(all_cols[0], all_cols[1], groups=groups)

#predefined splits
#gs = GridSearchCV(gb, grid, cv=ps.split(),scoring='accuracy')
gs = GridSearchCV(gb, grid, cv=ps.split(),scoring=['roc_auc','average_precision'],refit='roc_auc')

gs.fit(all_cols[0], all_cols[1])
print(gs.best_score_) #best cv score
print(gs.best_params_)
df_gridsearch = pd.DataFrame(gs.cv_results_)

df_gridsearch.to_csv('model_hp_results/gradientboosting_seqonly_hp.csv')


text:
import pandas as pd
import seaborn as sns #visualisation
import matplotlib.pyplot as plt #visualisation
import numpy as np
import os.path
sns.set(color_codes=True)

process_candidates = os.path.exists('./datasets/candidates.output')

lowLevelReps = open('./datasets/bytecode.output').readlines()
lowLevelReps = pd.Series([x.rstrip('\n') for x in lowLevelReps])
highLevelReps = open('./datasets/references.output').readlines()
highLevelReps = pd.Series([x.rstrip('\n') for x in highLevelReps])

df_pairs = pd.DataFrame({
        'source': lowLevelReps,
        'source_tokens': [len(x.split()) for x in lowLevelReps],
        'target' : highLevelReps,
        'target_tokens': [len(x.split()) for x in highLevelReps]
    })

if process_candidates:
    candidates = open('./datasets/candidates.output').readlines()
    candidates = [x.rstrip('\n') for x in candidates]
    df_pairs['candidate'] = candidates
    df_pairs['candidate_tokens'] = [len(x.split()) for x in candidates]

print(df_pairs.describe())

for x in range(10, 100, 5):
    q = df_pairs['source_tokens'].quantile(x/100)
    print("Quantile " + str(x) + "%: " + str(q))

mask = df_pairs['source_tokens'] < df_pairs['source_tokens'].quantile(0.95)
print ("Size of the dataset before filtering: " + str(len(df_pairs)))
df_pairs = df_pairs.loc[mask]
print ("Size of the dataset after filtering: " + str(len(df_pairs)))

#sns.boxplot(x=df_pairs['source_tokens'])
#sns.boxplot(x=df_pairs['target_tokens'])
sns.distplot( df_pairs['source_tokens'], bins=3 )
plt.show()

max_stars_repo_name: jmfinelli/JavaNeuralDecompiler

max_issues_repo_name: jmfinelli/JavaNeuralDecompiler

max_forks_repo_name: jmfinelli/JavaNeuralDecompiler


text:
from Bio import SeqIO
import numpy as np

path = "../data/sample.fastq"
# Guardar ids de records bajo umbral
mala_calidad = []
umbral = 32
for record in SeqIO.parse(path, 'fastq'):
    promedio = np.mean(record.letter_annotations['phred_quality'])
    if promedio < umbral:
        mala_calidad.append((promedio, record.id))

mala_calidad

# Eliminar si existe un valor menor al umbral
buena_calidad = []
umbral = 32
for record in SeqIO.parse(path, 'fastq'):
    minimo = min(record.letter_annotations['phred_quality'])
    if minimo > umbral:
        buena_calidad.append((record.id, record.seq))

buena_calidad

##########

for gb_record in SeqIO.parse("../data/aichi.gb", "genbank"):
    print("ID", gb_record.id)
    print('Secuencia', str(gb_record.seq)[0:30],'...')
    print('Longitud', len(gb_record))

# Informacion de metadatos almacenada como diccionario
gb_record.annotations

f_source = gb_record.features[0]
f_cds = gb_record.features[1]

print(f_source.location)
print(f_source.type)
print(f_source.qualifiers)

# Ejercicio 4

for virus in SeqIO.parse("../data/virus.gb", "genbank"):
    print(virus.annotations['sequence_version'])
    print(virus.annotations['organism'])
# Ejercicio 5

fuente_aislado = virus.features[0].qualifiers['isolation_source']
pais = virus.features[0].qualifiers['country']


text:
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import division
import os
import numpy as np
import soundfile as sf
import paddle.fluid.dygraph as dg

def make_output_tree(output_dir):
    checkpoint_dir = os.path.join(output_dir, "checkpoints")
    if not os.path.exists(checkpoint_dir):
        os.makedirs(checkpoint_dir)

    state_dir = os.path.join(output_dir, "states")
    if not os.path.exists(state_dir):
        os.makedirs(state_dir)

def valid_model(model, valid_loader, writer, global_step, sample_rate):
    loss = []
    wavs = []
    model.eval()
    for i, batch in enumerate(valid_loader):
        # print("sentence {}".format(i))
        audio_clips, mel_specs, audio_starts = batch
        y_var = model(audio_clips, mel_specs, audio_starts)
        wav_var = model.sample(y_var)
        loss_var = model.loss(y_var, audio_clips)
        loss.append(loss_var.numpy()[0])
        wavs.append(wav_var.numpy()[0])

    average_loss = np.mean(loss)
    writer.add_scalar("valid_loss", average_loss, global_step)
    for i, wav in enumerate(wavs):
        writer.add_audio("valid/sample_{}".format(i), wav, global_step,
                         sample_rate)

def eval_model(model, valid_loader, output_dir, global_step, sample_rate):
    model.eval()
    for i, batch in enumerate(valid_loader):
        # print("sentence {}".format(i))
        path = os.path.join(output_dir,
                            "sentence_{}_step_{}.wav".format(i, global_step))
        audio_clips, mel_specs, audio_starts = batch
        wav_var = model.synthesis(mel_specs)
        wav_np = wav_var.numpy()[0]
        sf.write(path, wav_np, samplerate=sample_rate)
        print("generated {}".format(path))

max_issues_repo_name: xbsdsongnan/Parakeet

max_forks_repo_name: xbsdsongnan/Parakeet


text:
import cv2
import numpy as np

class Warper:
    def __init__(self):
        h = 448
        w = 800
        print("h : " ,h)
        print("w : " ,w)

        # distort scr to dst
        # src = np.float32([
        #     [w * 1.6, h * 1.3],
        #     [w * (-0.1), h * 1.3],
        #     [0, h * 0.62],
        #     [w, h * 0.62],
        # ])
        src = np.float32([
            [480*1.6, 530*1.3],
            [480*(-0.1), 530*1.3],
            [0, h*0.62],
            [w, h*0.62],
        ])
        dst = np.float32([
            [w * 0.65, h],
            [w * 0.35, h],
            [-300, 0],
            [w+300, 0],
        ])
        # dst = np.float32([
        #     [0, 0],
        #     [w, 0],
        #     [0, h],
        #     [w , h],
        # ])

        self.M = cv2.getPerspectiveTransform(src, dst)
        self.Minv = cv2.getPerspectiveTransform(dst, src)

    def warp(self, img):
        return cv2.warpPerspective(
            img,
            self.M,
            (img.shape[1], img.shape[0]),
            flags=cv2.INTER_LINEAR
        )

    def unwarp(self, img):
        return cv2.warpPersective(
            img,
            self.Minv,
            (img.shape[1], img.shape[0]),
            flags=cv2.INTER_LINEAR
        )


text:
"""Make a panel figure for the paper of chromatic biases (RbarSqr, V, S_m02) vs r-i, both before
and after SVR correction.
"""

import cPickle
from argparse import ArgumentParser

import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import matplotlib.cm

band = 'LSST_r'
band_dict = {'LSST_r':"r",
             'LSST_i':"i"}
fontsize=12

# Some annotation arrow properties
arrowdict = dict(facecolor='black', shrink=0.1, width=1.5, headwidth=4, frac=0.2)

# hardcode some requirements, order is [DES, LSST]
r2sqr_gal = np.r_[0.4, 0.3]**2
r2sqr_PSF = np.r_[0.8, 0.7]**2

mean_m_req = np.r_[0.008, 0.003]
mean_DeltaRbarSqr_req = mean_m_req / 2.0
mean_DeltaV_req = r2sqr_gal * mean_m_req
mean_dS_m02_req = mean_m_req * r2sqr_gal / r2sqr_PSF

def set_range(x):
    """ Return a plotting range 30% larger than the interval containing 99% of the data.
    """
    xs = sorted(x)
    n = len(xs)
    low = xs[int(0.005*n)]
    high = xs[int(0.995*n)]
    span = high-low
    return [low - 0.3*span, high + 0.3*span]

def plot_panel(ax, galdata, colordata, cdata, cbands, ylabel, ylim):
    clim = [0.0, 2.2]

    rorder = np.random.permutation(len(colordata))

    cmap = matplotlib.cm.get_cmap('jet')
    im = ax.scatter(colordata[rorder], galdata[rorder], c=cdata[rorder],
                    vmin=clim[0], vmax=clim[1], zorder=4, s=3, cmap=cmap)
    im.set_rasterized(True)

    ax.set_xlabel(r"${} - {}$".format(band_dict[cbands[0]], band_dict[cbands[1]]),
                  fontsize=fontsize)
    ax.set_ylabel(ylabel, fontsize=fontsize)

    ax.set_ylim(ylim)
    for label in ax.get_xticklabels():
        label.set_fontsize(fontsize)
    for label in ax.get_yticklabels():
        label.set_fontsize(fontsize)

    return im

def bias_vs_color_panel(gals, stars, band, cbands, outfile):
    f, axarr = plt.subplots(3, 2, figsize=(9, 11))

    colordata = gals['mag'][cbands[0]] - gals['mag'][cbands[1]]
    cdata = gals['redshift']

    # RbarSqr
    ylabel = r"$\left(\Delta \overline{\mathrm{R}}\right)^2$ (arcsec$^2$)"
    stardata = stars['Rbar'][band] * 180/np.pi * 3600
    galdata = gals['Rbar'][band] * 180/np.pi * 3600
    norm = np.mean(stardata)
    stardata -= norm
    galdata -= norm
    stardata **= 2
    galdata **= 2
    ylim = set_range(galdata)
    ylim[0] = 0.0
    plot_panel(axarr[0, 0], galdata, colordata, cdata, cbands, ylabel, ylim)
    ax = axarr[0, 0]
    xlim = ax.get_xlim()
    ax.set_xlim(xlim)
    ax.fill_between(xlim, [ylim[0]]*2, [ylim[1]]*2, color='#BBBBBB')
    ax.fill_between(xlim, [0.0]*2, [mean_DeltaRbarSqr_req[0]]*2, color='#999999')
    ax.fill_between(xlim, [0.0]*2, [mean_DeltaRbarSqr_req[1]]*2, color='#777777')
    ax.axhline(-mean_DeltaRbarSqr_req[0], c='k', alpha=0.1, zorder=10, lw=0.5)
    ax.axhline(-mean_DeltaRbarSqr_req[1], c='k', alpha=0.3, zorder=10, lw=0.5)
    ax.axhline(mean_DeltaRbarSqr_req[0], c='k', alpha=0.1, zorder=10, lw=0.5)
    ax.axhline(mean_DeltaRbarSqr_req[1], c='k', alpha=0.3, zorder=10, lw=0.5)

    # Corrected RbarSqr
    stardata = (stars['Rbar'][band] - stars['photo_Rbar'][band]) * 180/np.pi * 3600
    galdata = (gals['Rbar'][band] - gals['photo_Rbar'][band]) * 180/np.pi * 3600
    # d((DR)^2) = 2 DR d(DR)
    stardata = 2 * (stars['Rbar'][band] * 180/np.pi * 3600 - norm) * stardata
    galdata = 2 * (gals['Rbar'][band] * 180/np.pi * 3600 - norm) * galdata
    ylabel = r"$\delta(\left(\Delta \overline{\mathrm{R}}\right)^2)$ (arcsec$^2$)"
    plot_panel(axarr[0, 1], galdata, colordata, cdata, cbands, ylabel, ylim)
    ax = axarr[0, 1]
    ax.set_xlim(xlim)
    ax.fill_between(xlim, [ylim[0]]*2, [ylim[1]]*2, color='#BBBBBB')
    ax.fill_between(xlim, [0.0]*2, [mean_DeltaRbarSqr_req[0]]*2, color='#999999')
    ax.fill_between(xlim, [0.0]*2, [mean_DeltaRbarSqr_req[1]]*2, color='#777777')
    ax.axhline(-mean_DeltaRbarSqr_req[0], c='k', alpha=0.1, zorder=10, lw=0.5)
    ax.axhline(-mean_DeltaRbarSqr_req[1], c='k', alpha=0.3, zorder=10, lw=0.5)
    ax.axhline(mean_DeltaRbarSqr_req[0], c='k', alpha=0.1, zorder=10, lw=0.5)
    ax.axhline(mean_DeltaRbarSqr_req[1], c='k', alpha=0.3, zorder=10, lw=0.5)

    # V
    ylabel = r"$\Delta \mathrm{V}}$ (arcsec$^2$)"
    stardata = stars['V'][band] * (180/np.pi * 3600)**2
    galdata = gals['V'][band] * (180/np.pi * 3600)**2
    norm = np.mean(stardata)
    stardata -= norm
    galdata -= norm
    ylim = set_range(np.concatenate([stardata, galdata]))
    plot_panel(axarr[1, 0], galdata, colordata, cdata, cbands, ylabel, ylim)
    ax = axarr[1, 0]
    xlim = ax.get_xlim()
    ax.set_xlim(xlim)
    ax.fill_between(xlim, [ylim[0]]*2, [ylim[1]]*2, color='#BBBBBB')
    ax.fill_between(xlim, [-mean_DeltaV_req[0]]*2, [mean_DeltaV_req[0]]*2, color='#999999')
    ax.fill_between(xlim, [-mean_DeltaV_req[1]]*2, [mean_DeltaV_req[1]]*2, color='#777777')
    ax.axhline(-mean_DeltaV_req[0], c='k', alpha=0.1, zorder=10, lw=0.5)
    ax.axhline(-mean_DeltaV_req[1], c='k', alpha=0.3, zorder=10, lw=0.5)
    ax.axhline(mean_DeltaV_req[0], c='k', alpha=0.1, zorder=10, lw=0.5)
    ax.axhline(mean_DeltaV_req[1], c='k', alpha=0.3, zorder=10, lw=0.5)
    ax.annotate("LSST requirement",
                xy=(-0.35, -mean_DeltaV_req[1]),
                xytext=(-0.25, -mean_DeltaV_req[1]-0.0003),
                arrowprops=arrowdict,
                zorder=10)
    ax.annotate("DES requirement",
                xy=(-0.35, -mean_DeltaV_req[0]),
                xytext=(-0.25, -mean_DeltaV_req[0]-0.0003),
                arrowprops=arrowdict,
                zorder=10)

    # Corrected V
    stardata = (stars['V'][band] - stars['photo_'+'V'][band]) * (180/np.pi * 3600)**2
    galdata = (gals['V'][band] - gals['photo_'+'V'][band]) * (180/np.pi * 3600)**2
    ylabel = "$\delta(\Delta \mathrm{V})$ (arcsec$^2$)"
    plot_panel(axarr[1, 1], galdata, colordata, cdata, cbands, ylabel, ylim)
    ax = axarr[1, 1]
    ax.set_xlim(xlim)
    ax.fill_between(xlim, [ylim[0]]*2, [ylim[1]]*2, color='#BBBBBB')
    ax.fill_between(xlim, [-mean_DeltaV_req[0]]*2, [mean_DeltaV_req[0]]*2, color='#999999')
    ax.fill_between(xlim, [-mean_DeltaV_req[1]]*2, [mean_DeltaV_req[1]]*2, color='#777777')
    ax.axhline(-mean_DeltaV_req[0], c='k', alpha=0.1, zorder=10, lw=0.5)
    ax.axhline(-mean_DeltaV_req[1], c='k', alpha=0.3, zorder=10, lw=0.5)
    ax.axhline(mean_DeltaV_req[0], c='k', alpha=0.1, zorder=10, lw=0.5)
    ax.axhline(mean_DeltaV_req[1], c='k', alpha=0.3, zorder=10, lw=0.5)
    ax.annotate("LSST requirement",
                xy=(-0.35, -mean_DeltaV_req[1]),
                xytext=(-0.25, -mean_DeltaV_req[1]-0.0005),
                arrowprops=arrowdict,
                zorder=10)
    ax.annotate("DES requirement",
                xy=(-0.35, -mean_DeltaV_req[0]),
                xytext=(-0.25, -mean_DeltaV_req[0]-0.0005),
                arrowprops=arrowdict,
                zorder=10)

    # S
    ylabel = r"$\Delta r^2_\mathrm{PSF}/r^2_\mathrm{PSF}$"
    stardata = stars['S_m02'][band]
    galdata = gals['S_m02'][band]
    starmean = np.mean(stardata)
    stardata = (stardata - starmean)/starmean
    galdata = (galdata - starmean)/starmean
    ylim = set_range(np.concatenate([stardata, galdata]))
    plot_panel(axarr[2, 0], galdata, colordata, cdata, cbands, ylabel, ylim)
    ax = axarr[2, 0]
    xlim = ax.get_xlim()
    ax.set_xlim(xlim)
    ax.fill_between(xlim, [ylim[0]]*2, [ylim[1]]*2, color='#BBBBBB')
    ax.fill_between(xlim, [-mean_dS_m02_req[0]]*2, [mean_dS_m02_req[0]]*2, color='#999999')
    ax.fill_between(xlim, [-mean_dS_m02_req[1]]*2, [mean_dS_m02_req[1]]*2, color='#777777')
    ax.axhline(-mean_dS_m02_req[0], c='k', alpha=0.1, zorder=10, lw=0.5)
    ax.axhline(-mean_dS_m02_req[1], c='k', alpha=0.3, zorder=10, lw=0.5)
    ax.axhline(mean_dS_m02_req[0], c='k', alpha=0.1, zorder=10, lw=0.5)
    ax.axhline(mean_dS_m02_req[1], c='k', alpha=0.3, zorder=10, lw=0.5)
    ax.annotate("LSST requirement",
                xy=(-0.35, -mean_dS_m02_req[1]),
                xytext=(-0.25, -mean_dS_m02_req[1]-0.003),
                arrowprops=arrowdict,
                zorder=10)
    ax.annotate("DES requirement",
                xy=(-0.35, -mean_dS_m02_req[0]),
                xytext=(-0.25, -mean_dS_m02_req[0]-0.003),
                arrowprops=arrowdict,
                zorder=10)

    # Corrected S
    stardata = (stars['S_m02'][band] - stars['photo_'+'S_m02'][band]) / stars['photo_'+'S_m02'][band]
    galdata = (gals['S_m02'][band] - gals['photo_'+'S_m02'][band]) / gals['photo_'+'S_m02'][band]
    ylabel = "$\delta(\Delta r^2_\mathrm{PSF}/r^2_\mathrm{PSF})$"
    im = plot_panel(axarr[2, 1], galdata, colordata, cdata, cbands, ylabel, ylim)
    xlim = ax.get_xlim()
    ax = axarr[2, 1]
    ax.set_xlim(xlim)
    ax.fill_between(xlim, [ylim[0]]*2, [ylim[1]]*2, color='#BBBBBB')
    ax.fill_between(xlim, [-mean_dS_m02_req[0]]*2, [mean_dS_m02_req[0]]*2, color='#999999')
    ax.fill_between(xlim, [-mean_dS_m02_req[1]]*2, [mean_dS_m02_req[1]]*2, color='#777777')
    ax.axhline(-mean_dS_m02_req[0], c='k', alpha=0.1, zorder=10, lw=0.5)
    ax.axhline(-mean_dS_m02_req[1], c='k', alpha=0.3, zorder=10, lw=0.5)
    ax.axhline(mean_dS_m02_req[0], c='k', alpha=0.1, zorder=10, lw=0.5)
    ax.axhline(mean_dS_m02_req[1], c='k', alpha=0.3, zorder=10, lw=0.5)
    ax.annotate("LSST requirement",
                xy=(-0.35, -mean_dS_m02_req[1]),
                xytext=(-0.25, -mean_dS_m02_req[1]-0.003),
                arrowprops=arrowdict,
                zorder=10)
    ax.annotate("DES requirement",
                xy=(-0.35, -mean_dS_m02_req[0]),
                xytext=(-0.25, -mean_dS_m02_req[0]-0.003),
                arrowprops=arrowdict,
                zorder=10)

    # colorbar
    colorbar_axes_range = [0.86, 0.77, 0.017, 0.19]
    cbar_ax = f.add_axes(colorbar_axes_range)
    cbar = plt.colorbar(im, cax=cbar_ax)
    cbar_ax.set_ylabel("redshift", fontsize=fontsize)

    f.tight_layout()
    f.savefig(outfile, dpi=400)

if __name__ == '__main__':
    parser = ArgumentParser()
    parser.add_argument('--galfile', default = "output/corrected_galaxy_data.pkl",
                        help="input galaxy file. Default 'output/corrected_galaxy_data.pkl'")
    parser.add_argument('--starfile', default = "output/corrected_star_data.pkl",
                        help="input star file. Default 'output/corrected_star_data.pkl'")
    parser.add_argument('--band', default='LSST_r', nargs='?',
                        help="band of chromatic bias to plot (Default: 'LSST_r')")
    parser.add_argument('--color', default=['LSST_r', 'LSST_i'], nargs=2,
                        help="color to use for symbol color (Default: ['LSST_r', 'LSST_i'])")
    parser.add_argument('--outfile', default="output/bias_vs_color_panel.pdf", nargs='?',
                        help="output filename (Default: 'output/bias_vs_color_panel.pdf')")
    args = parser.parse_args()

    gals = cPickle.load(open(args.galfile))
    stars = cPickle.load(open(args.starfile))
    bias_vs_color_panel(gals, stars, args.band, args.color, args.outfile)

max_stars_repo_name: DarkEnergyScienceCollaboration/chroma

max_issues_repo_name: DarkEnergyScienceCollaboration/chroma

max_forks_repo_name: DarkEnergyScienceCollaboration/chroma


text:
import numpy as np

#dir_path = './results/results_semhash_twitter_sentiment140_10runs/'
dir_path = './results/results_trigram_hash_twitter_sentiment140_768_10runs_macro/'

runs = 10

for data_type in ["corr"]: #, "inc", "inc_with_corr_TestOnlyIncorrect"]:
    str_write = ''
    for dataset_name in ['sentiment140']:
        str_write += dataset_name + '\n'

        acc_avg = 0
        acc_max = 0
        acc_arr = []
        for run in range(1, runs + 1):
            acc = 0
            subdir_path = dir_path + '{}_run{}/'.format(data_type, run)
            filename = subdir_path + '{}_f1.txt.txt'.format(dataset_name)
            f = open(filename, 'r')
            lines = f.read().split('\n')
            # Get max acc from 1 run
            for l in lines:
                if l is not "":
                    l_split = l.split(': ')
                    l_acc = float(l_split[1])
                    print(l_acc)
                    if l_acc >= acc:
                        acc = l_acc
            # Get average acc
            acc_avg += acc
            acc_arr.append(acc)
            # Get max acc
            if acc >= acc_max:
                acc_max = acc
        # acc_avg /= runs
        acc_avg = np.mean(acc_arr)
        acc_std = np.std(acc_arr)
        acc_max = np.max(acc_arr)
        str_write += '  Avg-{}: {:.2f}\pm{:.2f}\n'.format(runs, acc_avg * 100, acc_std * 100)
        str_write += '  Best-{}: {:.2f}\n\n'.format(runs, acc_max * 100)
    f_out = open(dir_path + data_type, 'w')
    f_out.write(str_write)

max_stars_repo_name: gcunhase/StackedDeBERT

max_issues_repo_name: gcunhase/StackedDeBERT

max_forks_repo_name: gcunhase/StackedDeBERT


text:
from operator import mul
import numpy as np

import gym
from gym import error, spaces, utils
from gym.utils import seeding
from tensorflow.python.ops.gen_linalg_ops import Qr
from gym_maze.envs.maze_view_2d import MazeView2D
import itertools
from sklearn.neighbors import KDTree
import time

class MazeEnv(gym.Env):

    ACTION = ["N","S", "E", "W"]
    VISITED_TO_IDX = {"visited":16}

    def __init__(self, maze_file=None, maze_size=None, mode=None, enable_render=True,num_goals = 1,verbose = True,human_mode=False,
        random_start_pos = False,random_goal_pos=False,seed_num = None,
        fixed_goals = None, fixed_start_pos = None,visited_cells = []):

        self.visited_cells = visited_cells
        self.verbose = verbose
        self.viewer = None
        self.enable_render = enable_render
        self.num_goals = num_goals
        self.fixed_start_pos = fixed_start_pos
        self.fixed_goals = fixed_goals

        self.human_mode = human_mode
        self.chosen_goal = None
        self.random_start_pos = random_start_pos
        self.random_goal_pos = random_goal_pos

        self.seed_num=self.seed(seed_num)

        if maze_file:
            print("maze from file")
            self.maze_view = MazeView2D(maze_name="OpenAI Gym - Maze (%s)" % maze_file,
                                        maze_file_path=maze_file,
                                        screen_size=(640, 640),
                                        enable_render=enable_render,
                                        num_goals = self.num_goals,random_start_pos = random_start_pos,
                                        random_goal_pos= random_goal_pos,
                                        verbose = self.verbose,np_random=self.np_random,
                                         fixed_goals = fixed_goals,
                                        fixed_start_pos =fixed_start_pos)

        else:
            raise AttributeError("One must supply either a maze_file path (str) or the maze_size (tuple of length 2)")

        # Set random

        self.maze_size = self.maze_view.maze_size

        # forward or backward in each dimension
        self.action_space = spaces.Discrete(2*len(self.maze_size))
        # print('self.action_space: ', self.action_space)
        # observation is the x, y coordinate of the grid
        low = np.zeros(len(self.maze_size), dtype=int)
        high =  np.array(self.maze_size, dtype=int) - np.ones(len(self.maze_size), dtype=int)
        self.observation_space = spaces.Box(low, high, dtype=np.int64)

        # initial condition
        self.state = None
        self.steps_beyond_done = None

        self.final_points = []

        self.covered_cells = self.maze_view.maze.maze_cells.copy()

        self.reset()

        self.configure()

    def setNewEntrance(self,entrance):
        self.fixed_start_pos = entrance
        self.maze_view.fixed_start_pos = entrance
        self.maze_view.setEntrance ( np.array(entrance) )
        print('self.visited_cells: ', len(self.visited_cells))

    def setNewGoals(self,goals):
        # TODO make multi
        self.fixed_goals = goals
        self.maze_view.fixed_goals = goals
        self.maze_view.setGoal(  np.array(goals[0]) )

    def setVisitedCells(self,cells):
        self.visited_cells = cells

    def __del__(self):
        if self.enable_render is True:
            self.maze_view.quit_game()

    def configure(self, display=None):
        self.display = display

    def seed(self, seed):
        np.random.seed(seed=seed)
        self.np_random = np.random.seed(seed=seed)
        return [seed]

    def get_n_to_be_covered(self):
        n_total_cells = np.count_nonzero(self.covered_cells != 0)
        n_covered_cells = np.count_nonzero(self.covered_cells == MazeEnv.VISITED_TO_IDX["visited"])
        return n_total_cells - n_covered_cells

    def compute_reward(self,moved):
        reward = 0
        if(self.num_goals==1):
            # Single goal mode
            if not moved:
                return -0.5,False
            elif np.array_equal(self.maze_view.robot, self.maze_view.goal):
                # Found goal
                reward = 1000
                done = True
            else:
                #  Not found goal
                if self.maze_view.robot.tolist() in self.visited_cells: #NOTE: O(len(visCellList)) cost
                    # Already been here
                    reward = -1
                else:
                    # oth: should be lower than not moved case to encourage moving
                    reward = -0.5/(self.maze_size[0]*self.maze_size[1])
                done = False
        elif self.num_goals > 1:
            # Multiple goals mode
            if not moved:
                return -5,False

            for _goal in self.maze_view.goals:
                # print('self.maze_view.robot, _goal: ', self.maze_view.robot, _goal)
                if np.array_equal(self.maze_view.robot, _goal):
                    if(self.enable_render):
                        self.maze_view.color_visited_cell(_goal[0],_goal[1])
                    reward = 300
                    self.maze_view.goals.remove( _goal )
                    break

            if reward < 1 : # Not found any goal
                reward = -1/(self.maze_size[0]*self.maze_size[1])
                done = False
            if(self.maze_view.goals==[]):
                done = True
            else:
                done = False
        else:
            # I am doing covering
            reward = 0
            if moved and self.covered_cells[self.maze_view.robot[0],self.maze_view.robot[1]] == MazeEnv.VISITED_TO_IDX["visited"]:
                reward = -0.5
                # cell already covered
            elif moved:
                reward = 1
                # New cell covered
                self.covered_cells[self.maze_view.robot[0],self.maze_view.robot[1]] = MazeEnv.VISITED_TO_IDX["visited"]

            elif not moved: # Not found any goal
                reward = -0.1/(self.maze_size[0]*self.maze_size[1])
                done = False

            if(self.verbose):
                print('self.get_n_to_be_covered(): ', self.get_n_to_be_covered())
            if self.get_n_to_be_covered() == 0:
                done = True
            else:
                done = False

        return reward,done

    def step(self, action):
        # print('self.maze_view.goals: ', self.maze_view.goals)
        if isinstance(action, int) or isinstance(action, np.int64):
            if(self.human_mode):
                my_action = int(input("Action (0 N,1 S,2 E,3 W, -1 random):"))
                action = my_action if my_action != -1 else action

            # print('action: ', action)
            _moved = self.maze_view.move_robot(self.ACTION[action])
        else:
            print('type(action): ', type(action))
            _moved = self.maze_view.move_robot(action)

        reward, done = self.compute_reward(moved=_moved)

        self.state = [ int(x) for x in self.maze_view.robot]

        if(_moved):
            if(self.enable_render):
                self.maze_view.color_visited_cell(self.state[0],self.state[1])

        # print('trajectory: ', self.trajectory)
        # print('self.kdtrees: ', self.kdtrees)

        info = {"moved": _moved }

        # print('self.state: ', self.state,_moved)
        return self.maze_view.robot, reward, done, info

    def reset(self):
        if(self.num_goals > 1):
            self.maze_view.goals = self.maze_view.saved_goals.copy()
        elif(self.num_goals < 1):
            # reset covered_cells
            self.covered_cells = self.maze_view.maze.maze_cells.copy()

        self.steps_beyond_done = None
        self.done = False

        #Redraw entrance
        if(self.random_start_pos):
            self.state = self.maze_view.entrance
            self.maze_view.resetEntrance()
        elif (self.fixed_start_pos is not None):
            self.state = self.maze_view.entrance
        else:
            # self.state = np.zeros(2)
            raise Exception("Either start goal or random must be specified")

        if(self.enable_render and self.num_goals < 1 ):
                # Reset all cells
                self.maze_view.maze_layer.fill((0, 0, 0, 0,))

        self.maze_view.reset_robot(rend=self.enable_render)

        return self.state

    def is_game_over(self):
        return self.maze_view.game_over

    def render(self, mode="human", close=False):
        if close:
            self.maze_view.quit_game()

        return self.maze_view.update(mode)

    def set_render(flag,self):
        self.enable_render = flag
        self.maze_view.__enable_render=flag
        self.maze_view.setScreen()

max_stars_repo_name: GiuppoUni/MALRL

max_issues_repo_name: GiuppoUni/MALRL

max_forks_repo_name: GiuppoUni/MALRL


text:
# -*- coding: utf-8 -*-
"""
Created on Feb 2020
@author: Yinchu Li (11849188@mail.sustech.edu.cn)
"""

################################################
########        DATA  GENERATION        ########
################################################

from getConfig import gConfig
import os
import numpy as np
from PIL import Image
from matplotlib.path import Path
from sklearn.model_selection import train_test_split
from scipy.io import loadmat

class data_preprocessing(object):

    def inputData_1d(self, dataPath, data_file, num_samples, vec_dim, num_channels=1):
        train_data = np.zeros([num_samples, vec_dim, num_channels])
        loaded_data = loadmat(os.path.join(dataPath, data_file))["data_output"]
        train_data[:, :, 0] = loaded_data
        return train_data

    def label_1d(self, labelPath, label_file, num_samples, label_dim, num_channels=1):
        # load y_train (casingCon_vector)
        train_target = np.zeros([num_samples, label_dim, num_channels])
        curve_data = loadmat(os.path.join(labelPath, label_file))["casingCon_discrete"]
        # log
        data_CasingCurve = np.log10(curve_data[1:, :])
        # scale to [0, 1]
        data_CasingCurve /= np.max(data_CasingCurve)
        train_target[:, :, 0] = data_CasingCurve
        return train_target

    def inputData_2d(self, dataPath, data_file,
                     num_samples, im_dim, num_channels,
                     data_form='log+scale', subtract_pixel_mean=True):
        # load X_train (images)
        if data_form == 'image':
            images = np.zeros([num_samples, im_dim, im_dim, num_channels])
            for i, img_name in enumerate(os.listdir(dataPath)):
                image_path = os.path.join(dataPath, img_name)
                img = Image.open(image_path)
                img_data = np.asarray(img, np.uint8)
                images[i, :, :, :] = img_data

            train_data = images.astype(np.float32) / 255.
            # If subtract pixel mean is enabled
            if subtract_pixel_mean:
                train_data_mean = np.mean(train_data, axis=0)
                train_data -= train_data_mean

        if data_form == 'log+scale': # log amplitudes, scaled angles
            train_data = np.zeros([num_samples, im_dim, im_dim, num_channels])
            amp_data = loadmat(os.path.join(dataPath, data_file))["data_log_amp"]
            ang_data = loadmat(os.path.join(dataPath, data_file))["data_scaled_ang"]
            train_data[:, :, :, 0] = amp_data
            train_data[:, :, :, 1] = ang_data

        return train_data

    def label_2d(self, labelPath, label_file, num_samples,
                 label_dim, num_channels):
        shape_data = loadmat(os.path.join(labelPath, label_file))["ShapeCollect"]
        fracLoc = loadmat(os.path.join(labelPath, label_file))["fracLoc"]
        train_target = np.zeros([num_samples, label_dim, label_dim, num_channels])

        # verts.shape = (9, num_samples * 2)
        verts = np.append(shape_data, shape_data[0].reshape(1, shape_data.shape[1]), axis=0)
        dim1, dim2 = label_dim, label_dim
        # Create vertex coordinates for each grid cell...
        # (<0,0> is at the top left of the grid in this system)
        x, y = np.meshgrid(np.linspace(fracLoc[0][2], fracLoc[0][3], num=dim1), np.linspace(fracLoc[0][4], fracLoc[0][5], num=dim2))
        x, y = x.flatten(), y.flatten()

        points = np.vstack((x, y)).T

        for i in range(num_samples):
            poly_verts = verts[:, 2 * (i + 1): 2 * ((i + 1) + 1)] # + 1 to skip first Shape data (none)
            path = Path(poly_verts)
            grid = path.contains_points(points)
            data_FracturingShape = grid.reshape((dim2, dim1))
            train_target[i, :, :, 0] = data_FracturingShape.astype(np.float32)
        return train_target

    def Split(self, train_data, train_target):
        X_train, X_vail, y_train, y_vail = train_test_split(train_data,
                                                            train_target,
                                                            test_size=gConfig['splitsize_train_others'],
                                                            random_state=gConfig['seed'])
        X_vail, X_test, y_vail, y_test = train_test_split(X_vail, y_vail,
                                                          test_size=gConfig['splitsize_vail_test'],
                                                          random_state=gConfig['seed'])

        input_shape = X_train.shape[1:]
        return input_shape, (X_train, y_train), (X_vail, y_vail), (X_test, y_test)

reason: import numpy,from scipy


text:
import pytest

import numpy as np

class DataGenerator:

    def __init__(self, fs=100e3, n_iter=10, sample_duration=1):
        self.fs = fs
        self.n_iter = n_iter
        self.sample_duration = sample_duration
        self.n_samples = int(round(sample_duration * fs))

    def _iter(self):
        self.generated = []
        for i in range(self.n_iter):
            samples = np.random.uniform(size=self.n_samples)
            self.generated.append(samples)
            yield samples

    def iter_continuous(self):
        yield from self._iter()

    def iter_epochs(self, isi):
        for i, samples in enumerate(self._iter()):
            signal = {
                'signal': samples,
                'info': {
                    't0': isi * i,
                    'duration': self.sample_duration,
                    'metadata': {'frequency': 1e3, 'stim': i},
                }
            }
            yield [signal]

@pytest.fixture
def data_generator():
    return DataGenerator()

max_stars_repo_name: NCRAR/psiexperiment

max_issues_repo_name: NCRAR/psiexperiment

max_forks_repo_name: NCRAR/psiexperiment


text:
import time
import numpy as np
from matplotlib import pyplot as plt

def plot_events(trace, events_pos, peaks_pos, peak_ampls, half_rises):
    plt.ion()  # interactive

    fig = plt.figure()
    ax = fig.add_subplot(111)

    x = np.linspace(-(50*0.04), 0.04, 100)  # 2 secs
    y = x.copy()
    y.fill(0)
    line1, = ax.plot(x, y, 'b-')  # Returns a tuple of line objects, thus the comma

    cursors_y = (0, 0, 0)  # defaults
    cursors_x = (x[0], x[len(x)/2], x[-1])
    line2, = ax.plot(cursors_x, cursors_y, 'r*')

    for event_start, event_end, half_rise_t in zip(events_pos, peaks_pos, half_rises):
        event = trace[(event_start-50):(event_start+50)]  # REFACTOR: why 50
        relative_event_start = 50
        relative_half_rise_t = 50 + half_rise_t-event_start
        relative_event_end = 50 + event_end-event_start
        line1.set_ydata(event)
        line2.set_xdata( (relative_event_start, relative_half_rise_t, relative_event_end) )
        line2.set_ydata( (event[relative_event_start], event[relative_half_rise_t], event[relative_event_end]) )
        fig.canvas.draw()
        time.sleep(0.05)

def _plot_trace(trace, trace_label, events_pos, peaks_pos, color=None, x=None, ax=None, y_shift=0.2, marker_size=100):
    if events_pos is False:
        return
    events_pos = np.array(events_pos, dtype=np.int64)
    peaks_pos = np.array(peaks_pos, dtype=np.int64)
    plot_element = ax if ax is not None else plt

    if color is None:
        plot_element.plot(x, trace, label=trace_label)
    else:
        plot_element.plot(x, trace, label=trace_label, color=color)

    if not len(events_pos):
        return

    sampling = x[1] - x[0]
    plt_y = np.take(trace, events_pos)
    plot_element.scatter(events_pos * sampling, plt_y, color='g', s=marker_size, zorder=5, label='event start')
    plt_y = np.take(trace, peaks_pos) + y_shift
    plot_element.scatter(peaks_pos * sampling, plt_y, color='r', s=marker_size, zorder=4, label='event end')

    plot_element.grid(color='blue', linestyle='--')

max_stars_repo_name: Sepidak/spikeGUI

max_issues_repo_name: Sepidak/spikeGUI

max_forks_repo_name: Sepidak/spikeGUI


text:
import numpy as np
import prettytensor as pt
import tensorflow as tf

class BaselineTfImage(object):
    coeffs = None

    def __init__(self, session):
        self.net = None
        self.session = session

    def create_net(self, shape):
        self.x = tf.placeholder(tf.float32, shape=[None, shape[1], shape[2], shape[3]], name="x")
        self.y = tf.placeholder(tf.float32, shape=[None], name="y")
        self.net = (pt.wrap(self.x).
                    conv2d(1, 16, stride=2, batch_normalize=True).
                    conv2d(1, 16, stride=2, batch_normalize=True).
                    flatten().
                    fully_connected(32, activation_fn=tf.nn.relu).
                    fully_connected(32, activation_fn=tf.nn.relu).
                    fully_connected(1))
        self.net = tf.reshape(self.net, (-1, ))
        l2 = (self.net - self.y) * (self.net - self.y)
        self.train = tf.train.AdamOptimizer().minimize(l2)
        self.session.run(tf.initialize_all_variables())

    def _features(self, path):
        ret = path["observations"].astype('float32')
        return ret

    def fit(self, paths):
        featmat = np.concatenate([self._features(path) for path in paths])
        if self.net is None:
            self.create_net(featmat.shape)
        returns = np.concatenate([path["returns"] for path in paths])
        for _ in range(100):
            self.session.run(self.train, {self.x: featmat, self.y: returns})

    def predict(self, path):
        if self.net is None:
            return np.zeros(len(path["rewards"]))
        else:
            ret = self.session.run(self.net, {self.x: self._features(path)})
            return np.reshape(ret, (ret.shape[0], ))


text:
import os
import torch
import torch.nn as nn

import numpy as np
from imageio import imsave
from torchvision.utils import make_grid
import torch.nn.utils.prune as prune

import argparse
from tqdm import tqdm
from copy import deepcopy

from utils.inception_score import get_inception_score
from utils.fid_score import calculate_fid_given_paths
from utils.inception_score import _init_inception
from utils.fid_score import create_inception_graph, check_or_download_inception

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("--load-path", type=str)
    parser.add_argument('--bottom_width', type=int, default=4)
    parser.add_argument('--latent_dim', type=int, default=128)
    parser.add_argument('--gf_dim', type=int, default=256)
    parser.add_argument('--df_dim', type=int, default=128)
    parser.add_argument('--round', type=int, default=3)
    parser.add_argument('--eval_batch_size', type=int, default=100)
    parser.add_argument('--num_eval_imgs', type=int, default=50000)
    args = parser.parse_args()
    return args

class GenBlock(nn.Module):
    def __init__(self, in_channels, out_channels, hidden_channels=None, ksize=3, pad=1,
                 activation=nn.ReLU(), upsample=False, n_classes=0):
        super(GenBlock, self).__init__()
        self.activation = activation
        self.upsample = upsample
        self.learnable_sc = in_channels != out_channels or upsample
        hidden_channels = out_channels if hidden_channels is None else hidden_channels
        self.n_classes = n_classes
        self.c1 = nn.Conv2d(in_channels, hidden_channels, kernel_size=ksize, padding=pad)
        self.c2 = nn.Conv2d(hidden_channels, out_channels, kernel_size=ksize, padding=pad)

        self.b1 = nn.BatchNorm2d(in_channels)
        self.b2 = nn.BatchNorm2d(hidden_channels)
        if self.learnable_sc:
            self.c_sc = nn.Conv2d(in_channels, out_channels, kernel_size=1, padding=0)

    def upsample_conv(self, x, conv):
        return conv(nn.UpsamplingNearest2d(scale_factor=2)(x))

    def residual(self, x):
        h = x
        h = self.b1(h)
        h = self.activation(h)
        h = self.upsample_conv(h, self.c1) if self.upsample else self.c1(h)
        h = self.b2(h)
        h = self.activation(h)
        h = self.c2(h)
        return h

    def shortcut(self, x):
        if self.learnable_sc:
            x = self.upsample_conv(x, self.c_sc) if self.upsample else self.c_sc(x)
            return x
        else:
            return x

    def forward(self, x):
        return self.residual(x) + self.shortcut(x)

class Generator(nn.Module):
    def __init__(self, args, activation=nn.ReLU(), n_classes=0):
        super(Generator, self).__init__()
        self.bottom_width = args.bottom_width
        self.activation = activation
        self.n_classes = n_classes
        self.ch = args.gf_dim
        self.l1 = nn.Linear(args.latent_dim, (self.bottom_width ** 2) * self.ch)
        self.block2 = GenBlock(self.ch, self.ch, activation=activation, upsample=True, n_classes=n_classes)
        self.block3 = GenBlock(self.ch, self.ch, activation=activation, upsample=True, n_classes=n_classes)
        self.block4 = GenBlock(self.ch, self.ch, activation=activation, upsample=True, n_classes=n_classes)
        self.b5 = nn.BatchNorm2d(self.ch)
        self.c5 = nn.Conv2d(self.ch, 3, kernel_size=3, stride=1, padding=1)

    def forward(self, z):

        h = z
        h = self.l1(h).view(-1, self.ch, self.bottom_width, self.bottom_width)
        h = self.block2(h)
        h = self.block3(h)
        h = self.block4(h)
        h = self.b5(h)
        h = self.activation(h)
        h = nn.Tanh()(self.c5(h))
        return h

def pruning_generate(model, state_dict):

    parameters_to_prune =[]
    for (name, m) in model.named_modules():
        if isinstance(m, nn.Conv2d) or isinstance(m, nn.ConvTranspose2d):
            m = prune.custom_from_mask(m, name = 'weight', mask = state_dict[name + ".weight_mask"])

def see_remain_rate(model):
    sum_list = 0
    zero_sum = 0
    for m in model.modules():
        if isinstance(m, nn.Conv2d) or isinstance(m, nn.ConvTranspose2d):
            sum_list = sum_list+float(m.weight.nelement())
            zero_sum = zero_sum+float(torch.sum(m.weight == 0))
    print('remain weight = ', 100*(1-zero_sum/sum_list),'%')

def see_remain_rate_orig(model):
    sum_list = 0
    zero_sum = 0
    for m in model.modules():
        if isinstance(m, nn.Conv2d) or isinstance(m, nn.ConvTranspose2d):
            sum_list = sum_list+float(m.weight_orig.nelement())
            zero_sum = zero_sum+float(torch.sum(m.weight_orig == 0))
    print('remain weight = ', 100*(1-zero_sum/sum_list),'%')

def see_remain_rate_mask(model):
    sum_list = 0
    zero_sum = 0
    for m in model.modules():
        if isinstance(m, nn.Conv2d) or isinstance(m, nn.ConvTranspose2d):
            sum_list = sum_list+float(m.weight_orig.nelement())
            zero_sum = zero_sum+float(torch.sum(m.weight_mask == 0))
    print('remain weight = ', 100*(1-zero_sum/sum_list),'%')

def evaluate(args, fixed_z, fid_stat, gen_net: nn.Module):
    # eval mode
    gen_net = gen_net.eval()

    # generate images
    sample_imgs = gen_net(fixed_z)
    img_grid = make_grid(sample_imgs, nrow=5, normalize=True, scale_each=True)

    # get fid and inception score
    fid_buffer_dir = 'fid_buffer_test'
    if not os.path.exists(fid_buffer_dir): os.makedirs(fid_buffer_dir)

    eval_iter = args.num_eval_imgs // args.eval_batch_size
    img_list = list()
    for iter_idx in tqdm(range(eval_iter), desc='sample images'):
        z = torch.cuda.FloatTensor(np.random.normal(0, 1, (args.eval_batch_size, args.latent_dim)))

        # Generate a batch of images
        gen_imgs = gen_net(z).mul_(127.5).add_(127.5).clamp_(0.0, 255.0).permute(0, 2, 3, 1).to('cpu', torch.uint8).numpy()
        for img_idx, img in enumerate(gen_imgs):
            file_name = os.path.join(fid_buffer_dir, 'iter%d_b%d.png' % (iter_idx, img_idx))
            imsave(file_name, img)
        img_list.extend(list(gen_imgs))

    # get inception score
    print('=> calculate inception score')
    mean, std = get_inception_score(img_list)

    # get fid score
    print('=> calculate fid score')
    fid_score = calculate_fid_given_paths([fid_buffer_dir, fid_stat], inception_path=None)

    os.system('rm -r {}'.format(fid_buffer_dir))
    return mean, fid_score

def main():
    args = parse_args()
    gen_net = Generator(args).cuda()
    _init_inception()
    inception_path = check_or_download_inception(None)
    create_inception_graph(inception_path)
    assert os.path.exists(args.load_path), "checkpoint file {} is not found".format(args.load_path)
    checkpoint = torch.load(args.load_path)
    torch.manual_seed(12345)
    torch.cuda.manual_seed(12345)
    np.random.seed(12345)
    #print("remaining percent: {}".format(0.8 ** checkpoint['round']))
    pruning_generate(gen_net, checkpoint['avg_gen_state_dict']) # Create a buffer for mask]
    gen_net.load_state_dict(checkpoint['avg_gen_state_dict'])

    see_remain_rate_mask(gen_net)
    see_remain_rate(gen_net)
    print("Best FID:{}".format(checkpoint['best_fid']))
    fixed_z = torch.cuda.FloatTensor(np.random.normal(0, 1, (25, args.latent_dim)))
    fid_stat = 'fid_stat/fid_stats_cifar10_train.npz'
    inception_score, fid_score = evaluate(args, fixed_z, fid_stat, gen_net)

    print('Inception score: %.4f, FID score: %.4f' % (inception_score, fid_score))

if __name__ == '__main__':
    main()