text:
"""Benchmarks for polynomials over Galois fields. """

from __future__ import print_function, division

from sympy.polys.galoistools import gf_from_dict, gf_factor_sqf
from sympy.polys.domains import ZZ
from sympy import pi, nextprime

def gathen_poly(n, p, K):
    return gf_from_dict({n: K.one, 1: K.one, 0: K.one}, p, K)

def shoup_poly(n, p, K):
    f = [K.one] * (n + 1)
    for i in range(1, n + 1):
        f[i] = (f[i - 1] ** 2 + K.one) % p
    return f

def genprime(n, K):
    return K(nextprime(int((2 ** n * pi).evalf())))

p_10 = genprime(10, ZZ)
f_10 = gathen_poly(10, p_10, ZZ)

p_20 = genprime(20, ZZ)
f_20 = gathen_poly(20, p_20, ZZ)

def timeit_gathen_poly_f10_zassenhaus():
    gf_factor_sqf(f_10, p_10, ZZ, method="zassenhaus")

def timeit_gathen_poly_f10_shoup():
    gf_factor_sqf(f_10, p_10, ZZ, method="shoup")

def timeit_gathen_poly_f20_zassenhaus():
    gf_factor_sqf(f_20, p_20, ZZ, method="zassenhaus")

def timeit_gathen_poly_f20_shoup():
    gf_factor_sqf(f_20, p_20, ZZ, method="shoup")

P_08 = genprime(8, ZZ)
F_10 = shoup_poly(10, P_08, ZZ)

P_18 = genprime(18, ZZ)
F_20 = shoup_poly(20, P_18, ZZ)

def timeit_shoup_poly_F10_zassenhaus():
    gf_factor_sqf(F_10, P_08, ZZ, method="zassenhaus")

def timeit_shoup_poly_F10_shoup():
    gf_factor_sqf(F_10, P_08, ZZ, method="shoup")

def timeit_shoup_poly_F20_zassenhaus():
    gf_factor_sqf(F_20, P_18, ZZ, method="zassenhaus")

def timeit_shoup_poly_F20_shoup():
    gf_factor_sqf(F_20, P_18, ZZ, method="shoup")

max_stars_repo_name: msgoff/sympy

max_issues_repo_name: msgoff/sympy

max_forks_repo_name: msgoff/sympy

reason: from sympy


text:
import pytest
import numpy as np
import pandas as pd

from toad.selection import drop_empty, drop_var, drop_corr, drop_iv, drop_vif, select, stepwise

from generate_data import frame

def test_drop_empty():
    df = drop_empty(frame, threshold = 0.8)
    assert 'E' not in df

def test_drop_var():
    df = drop_var(frame, threshold = 0.1)
    assert 'A' not in df

def test_drop_var_exclude():
    df = drop_var(frame, threshold = 0.1, exclude = 'A')
    assert 'A' in df

def test_drop_corr():
    df = drop_corr(frame, target = 'target')
    assert ['D', 'E', 'F', 'target'] == df.columns.tolist()

def test_drop_iv():
    df = drop_iv(frame, target = 'target', threshold = 0.25)
    assert 'B' not in df

def test_select():
    df = select(frame, target = 'target', empty = 0.8, iv = 0.2, corr = 0.7)
    assert ['D', 'F', 'target'] == df.columns.tolist()

def test_select_exclude():
    df = select(frame, target = 'target', empty = 0.8, iv = 0.2, corr = 0.7, exclude = ['A'])
    assert ['A', 'D', 'F', 'target'] == df.columns.tolist()

def test_stepwise():
    df = stepwise(frame.fillna(-1), target = 'target')
    assert ['C', 'E', 'F', 'target'] == df.columns.tolist()

def test_stepwise_backward():
    df = stepwise(frame.fillna(-1), target = 'target', direction = 'backward')
    assert ['C', 'E', 'F', 'target'] == df.columns.tolist()

def test_stepwise_forward():
    df = stepwise(frame.fillna(-1), target = 'target', direction = 'forward')
    assert ['C', 'E', 'F', 'target'] == df.columns.tolist()

def test_stepwise_exclude():
    df = stepwise(frame.fillna(-1), target = 'target', exclude = 'B')
    assert ['B', 'C', 'E', 'F', 'target'] == df.columns.tolist()

def test_stepwise_return_drop():
    df, drop_list = stepwise(frame.fillna(-1), target = 'target', return_drop = True)
    assert ['B', 'A', 'D'] == drop_list

def test_stepwise_lr():
    df = stepwise(frame.fillna(-1), target = 'target', estimator = 'lr', direction = 'forward')
    assert ['A', 'B', 'C', 'E', 'target'] == df.columns.tolist()

def test_drop_vif():
    df = drop_vif(frame.fillna(-1), exclude = 'target')
    assert ['C', 'F', 'target'] == df.columns.tolist()

ext: py

max_stars_repo_name: flysaint/toad

max_issues_repo_name: flysaint/toad

max_forks_repo_name: flysaint/toad

reason: import numpy


text:
"""
Script to combine a FIRMS export and a USDA weather station export

This will provide insight to the conditions at each detection location including soil moisture, relative humidity, etc.

FIRMS (Fire Information Resource Management System) exports of satellite fire dections can be optained in the following link:
https://firms.modaps.eosdis.nasa.gov/download/

USDA report exports of stations that track soil moisture, wind, humidity etc. can be found at the folowing link:
https://wcc.sc.egov.usda.gov/reportGenerator/
Note that the export comes with some headers at the top of the csv you have to manually remove first

A map of the stations can be found here:
https://www.nrcs.usda.gov/wps/portal/wcc/home/quicklinks/imap

Using the python module 'geopy' for calculating distance between the detection and the USDA weather stations
before running install the geopy module:
pip install geopy

You will want to change the file names and paths for merging the data with your own FIRMS and USDA exports

BA_
"""

import pandas as pd
import numpy as np
import geopy.distance

#import the 2 csvsand merge them.
#TODO FOR RUNNING THIS ON OWN: you will have to change the csv location/name
dfa = pd.read_csv(r'C:\Users\anderb4\Documents\GeorgetownDSCert\WildfireCapstone\Data\Modis20to22\fire_archive_M-C61_258142.csv')
dfb = pd.read_csv(r'C:\Users\anderb4\Documents\GeorgetownDSCert\WildfireCapstone\Data\Modis20to22\fire_nrt_M-C61_258142.csv')
df = pd.concat([dfa, dfb]) #concatonate the data

#get a rounded longitude and latitude  in the FIRMS df
df['lat'] = round(df['latitude'])
df['long'] = round(df['longitude'])
#DFB below is for testing delete before releasing full
dfb['lat'] = round(dfb['latitude'])
dfb['long'] = round(dfb['longitude'])

#now read the USDA exports
#TODO FOR DOING THIS ON OWN: Chan ge file name/path to your own USDA export
df_SCAN2020 = pd.read_csv('SCANSelectedData2020.csv')
df_SCAN2021 = pd.read_csv('SCANSelectedData2021.csv')
dfSCAN = pd.concat([df_SCAN2020, df_SCAN2021]) #concatonate the scan data

#get a dataframe only of each unique station
StationDFCol = dfSCAN.columns[1:14] #list of columns relevant for the station
dfStation = dfSCAN[StationDFCol] #get a dataframe of just the station info
dfStation = dfStation.drop_duplicates() #remove redundant rows so now just a new df of station info

#convert both the FIRMS and USDA to date time object
df['newdate'] = pd.to_datetime(df['acq_date'], format = '%Y-%m-%d') #converts the df to date time
def ReplaceSlash(datein):
    dashdate = datein.replace('/', '-')
    return dashdate
dfSCAN['date_new'] = dfSCAN['Date'].apply(ReplaceSlash) #converts the format
dfSCAN['date_new'] =  pd.to_datetime(dfSCAN['date_new'], format = '%m-%d-%Y') #converts the df to date time
#Both dfs now have a date in pandas date time format

def CalcDist(lat1, long1, lat2, long2):
    """
    Uses geopy python module
    Function which gets distance between any 2 coordinates
    Returns distance in miles
    """
    coords_1 = (lat1, long1)
    coords_2 = (lat2, long2)
    dist = geopy.distance.geodesic(coords_1, coords_2).miles
    return dist

def CalcDistDF(row):
    """
    Same function as calc dist but for a row in a df
    """
    stationLat = row['Latitude']
    stationLong = row['Longitude']
    detectionLat = row['latcompare']
    detectionLong = row['longcompare']
    calcDist = CalcDist(stationLat, stationLong, detectionLat, detectionLong)
    return (calcDist)

#attach nearest station to each detection
df['nearestStation'] = '' #nearest station name
df['StationDist'] = np.nan #will be distance between the detection and nearest station
dfTest2 = df.iloc[:1000] #create a smaller test df
progressCounter = 0
for index, i in df.iterrows():
    tmp = dfStation #creates a temp station df
    tmp['latcompare'] = i['latitude'] #enters the detection point lat in temp df
    tmp['longcompare'] = i['longitude'] #enters the detection point lat in temp df
    tmp['CalcDist'] = tmp.apply(lambda row: CalcDistDF(row), axis=1) #creates a column of distance between all stations and this point
    tmp = tmp.sort_values(by='CalcDist') #sorts the calculated distances with closest on top
    topStation = tmp.iloc[0] #gets the top station
    df.at[index, 'nearestStation'] = topStation['Station Name'] #populates the nearest station name
    df.at[index, 'StationDist'] = topStation['CalcDist'] #populates how far away the station is
    if progressCounter%10000 == 0:
        print('on line: ' + str(progressCounter)) #print the progress every 10000
    progressCounter = progressCounter + 1

#the nearest station is now in the FIRMS dataframe
#now merge the SCAN datframe by station and date to the FIRMS dataframe
dfMerge = df.merge(dfSCAN, left_on=['nearestStation', 'newdate'], right_on=['Station Name', 'date_new'], how='left')

#dfMerge.to_csv('FIRMSandSCAN20To21_Try4.csv') #TODO name the csv you wanty to save

"""
This ran successfully for me and output the csv above with the merged USDA data with each FIRMS detection

The merged data includes soil moisture, relative humidity, average wind speed, etc.

"""

max_stars_repo_name: georgetown-analytics/wildfires

max_issues_repo_name: georgetown-analytics/wildfires

max_forks_repo_name: georgetown-analytics/wildfires


text:
import numpy as np
import pandas as pd
import geopandas
import json
from shapely.geometry import MultiPoint
from tqdm import tqdm

def read_geojson_with_power(path_to_geojson):
    """Reads a GeoJSON with a list in one of the features

    GeoPandas does not import lists from GeoJSON.
    We need a power list for each charging point to calculate the charging time

    Args:
        path_to_geojson (string): Path to file.

    Returns:
        df: Pandas dataframe containing all features as columns with respective data per entry
    """
    with open(path_to_geojson, "r") as infile:
        inp = json.load(infile)
    # iterate over entries
    power, rest, geom = [], [], []
    for feat in inp["features"]:
        power.append(feat["properties"]["power"])
        geometry = MultiPoint([feat["geometry"]["coordinates"][0]])
        geom.append(geometry)
        props = feat["properties"]
        rest.append([props["max_ASP_FZG"], props["max_ASP_PW"],
                    props["max_ASP_LI"], props["max_ASP_LW"], props["max_ASP_LZ"]])
    rest = np.array(rest)
    df = pd.DataFrame(rest, columns=[
                      "max_ASP_FZG", "max_ASP_PW", "max_ASP_LI", "max_ASP_LW", "max_ASP_LZ"])
    df["geometry"] = geom
    df["power"] = power
    return df

def minimizer(file, pct, what, column):
    """Minimizes the numbers in column of df by pct

    We use it to calculate how many cars are e-cars or e-cars charging.

    Args:
        file (df): A dataframe with column.
        pct (float/int): Percentage to multiply with column.
        what (str): Name of new column in df.
        column (str): Name of column in df to minimize.

    Returns:
        None
    """
    liste = []
    for i in range(len(file)):
        EAutos = file[column][i] * pct
        liste.append(EAutos)
    file[what] = liste

def charging(tankstellen, timeframe, consumption, winter, when, driven, bat_cap):
    """Calculate how many e-cars are charging based on params

    Args:
        tankstellen (gdf): Df containing all charging points with respective load (Belastung)
        winter (float): Worstcase winter, multiplicator how much capacity will be left in cold weather (0.8 => 80%)
        when (float): People charge not at zero percent state of charge
        driven (float): How many km people drive in a day by car
        bat_cap (int): Battery capacity in kWh
        consumption (int/float): Consumption of average e-car in kwh/100km

    Returns:
        None
    """
    dist = 100 * bat_cap / consumption  # 100 * kwh / (kwh / 100km)
    pct = driven / (dist * winter * when)

    minimizer(tankstellen, pct, "E-Autos_charging_" +
              timeframe, "E-Autos_" + timeframe)

def capacity(tankstellen, when, bat_cap, max_power):
    """Calculate how many e-cars can charge in an hour at each charging point with max given charging power

    When technology is quickly transformed in e-cars, the charging points will also get stronger.
    If you pass a average charging power of 250 kW, every charging point is calculated as it is having 250 kW, despite it may not be that strong atm

    Args:
        tankstellen (gdf): Df containing all charging points with respective load (Belastung)
        when (float): People charge not at zero percent state of charge
        bat_cap (int): Battery capacity in kWh
        max_power (int/float): Average charging power for a charging process in kW

    Returns:
        list_capacity (list): List containing capacity per charging point
    """
    list_capacity = []
    for i in range(len(tankstellen["power"])):
        summe = 0
        for j in range(len(tankstellen["power"][i])):
            if max_power >= 250:
                summe += max_power
            elif tankstellen["power"][i][j] > max_power:
                summe += max_power
            else:
                summe += tankstellen["power"][i][j]
        # charges only to 80% and was at given percentage (when)
        list_capacity.append(summe/(bat_cap * (0.8 - (1 - when))))

    return list_capacity

def sufficiency(tankstellen, scenario, capacity):
    """Calculate ratio if charging point is sufficient for e cars that want to charge.

    Args:
        tankstellen (gdf): Df containing all charging points with respective load (Belastung)
        scenario (string): Name of scenario column to calculate sufficiency
        capacity (string): Name of capacity column to calculate sufficiency

    Returns:
        list_sufficiency (list): List containing sufficiency per charging point based on capacity and load
    """
    list_sufficiency = []
    for j in range(len(tankstellen)):
        list_sufficiency.append(
            tankstellen[scenario][j]/tankstellen[capacity][j])

    return list_sufficiency

def weighter(cities, tankstellen, scenario, city_weight):
    """Weight charging points in 5km vicinity of coordinates in cities based on city_weight.

    If you weigh some areas more, the rest of the charging points will get less cars (distributed in whole Switzerland)

    Args:
        cities (gdf): Df containing coordinates of important areas (cities)
        tankstellen (gdf): Df containing all charging points with respective load (Belastung)
        scenario (string): Name of scenario column to calculate sufficiency
        city_weight (int/float): Weight for area (2 for twice as many cars charge here than rest)

    Returns:
        tankstellen (df): Dataframe which has updated load of each charging point based on weight
    """
    tankstellen = geopandas.GeoDataFrame(
        tankstellen, crs="EPSG:4326", geometry=tankstellen.geometry)  # Convert df to gdf to call .cx

    in_a_city = []
    for i in range(len(tankstellen)):  # init list base case NOT in a city
        in_a_city.append(0)

    for i in range(len(cities)):
        emin, nmin, emax, nmax = cities.e[i]-5000, cities.n[i] - \
            5000, cities.e[i]+5000, cities.n[i]+5000

        # only charging points in bbox are in city (1)
        charging_in_a_city = tankstellen.cx[emin:emax, nmin:nmax]
        for point in charging_in_a_city.iterrows():
            in_a_city[point[0]] = 1
    tankstellen["in_a_city"] = in_a_city

    return weighter_multiply(tankstellen, scenario, city_weight)

def weighter_multiply(tankstellen, scenario, city_weight=1):
    """Weight charging points in 5km vicinity of coordinates in cities based on city_weight.

    This is a helper function for weighter(), which multiplies the given charging points with the given weight (default 1).

    Args:
        tankstellen (gdf): Df containing all charging points with respective load (Belastung)
        scenario (string): Name of scenario column to calculate sufficiency
        city_weight (int/float): Weight for area (2 for twice as many cars charge here than rest)

    Returns:
        tankstellen (df): Dataframe which has updated load of each charging point based on weight
    """
    sum_charging_points_in_cities = tankstellen.in_a_city.sum()
    for i in range(len(tankstellen)):
        if tankstellen["in_a_city"][i] == 1:
            tankstellen[scenario][i] = tankstellen[scenario][i] * city_weight
        else:
            # print()
            tankstellen[scenario][i] = tankstellen[scenario][i] * (len(
                tankstellen) - city_weight * sum_charging_points_in_cities) / (len(tankstellen) - sum_charging_points_in_cities)
    return tankstellen

def sufficiency_pct(start, finish, step, tankstellen, cities, consumption, winter, when, driven, bat_cap, max_power):
    """Finds the number of sufficient (sufficiency <= 1) charging points with a percentage range of e cars on roads

    With this you can analyze how the model reacts to parameter changes with over a load range.

    Args:
        start (float): start of range
        finish (float): end of range
        step (float): step in range
        tankstellen (gdf): Df containing all charging points with respective load (Belastung)
        cities (gdf): Df containing coordinates of important areas (cities)
        winter (float): Worstcase winter, multiplicator how much capacity will be left in cold weather (0.8 => 80%)
        when (float): People charge not at zero percent state of charge
        driven (float): How many km people drive in a day by car
        bat_cap (int): Battery capacity in kWh
        consumption (int/float): Consumption of average e-car in kwh/100km
        max_power (int/float): Average charging power for a charging process in kW

    Returns:
        sufficiency_x_y (df): Dataframe which has the range in one column ("pct_of_e_cars_on_roads") and
                              the number of sufficient charging points in an other ("sufficiency")
    """
    range_list = np.arange(start / 100, finish / 100, step / 100).tolist()
    sufficiency_x_y = pd.DataFrame()
    sufficiency_x_y["pct_of_e_cars_on_roads"] = range_list
    sufficiency_per_pct = []

    for pct in range_list:
        minimizer(tankstellen, pct, "E-Autos_" + str(pct), 'max_ASP_PW')

    tankstellen["capacity_pct"] = capacity(
        tankstellen, when, bat_cap, max_power)

    for pct in tqdm(range_list):
        charging(tankstellen, str(pct), consumption,
                 winter, when, driven, bat_cap)
        tankstellen = weighter(cities, tankstellen,
                               "E-Autos_charging_" + str(pct), city_weight=2)
        tankstellen["sufficiency_" + str(pct)] = sufficiency(
            tankstellen, "E-Autos_charging_" + str(pct), "capacity_pct")
        sum = 0
        for i in range(len(tankstellen["sufficiency_" + str(pct)])):
            if tankstellen["sufficiency_" + str(pct)][i] < 1:
                sum += 1
        sufficiency_per_pct.append(sum)

    sufficiency_x_y["sufficiency"] = sufficiency_per_pct

    return sufficiency_x_y


text:
from typing import Callable

import cv2
import numpy as np
from torch.utils.data import Dataset
from torchvision import transforms as transforms

class TransformDataset(Dataset):

    def __init__(self, dataset: Dataset, transform: Callable):
        self.dataset = dataset
        self.transform = transform

    def __getitem__(self, index: int):
        dataPoint = self.dataset[index]
        if isinstance(dataPoint, tuple):
            return (self.transform(dataPoint[0]),) + dataPoint[1:]
        else:
            return self.transform(dataPoint)

    def __len__(self):
        return len(self.dataset)

class PairTransform(object):
    def __init__(self, transform):
        self.transform = transform

    def __call__(self, sample):
        xi = self.transform(sample)
        xj = self.transform(sample)
        return xi, xj

def get_simclr_image_transform(color_scale, input_shape):
    # get a set of data augmentation transformations as described in the SimCLR paper.
    color_jitter = transforms.ColorJitter(0.8 * color_scale, 0.8 * color_scale, 0.8 * color_scale, 0.2 * color_scale)
    data_transforms = transforms.Compose([transforms.RandomResizedCrop(size=input_shape[0]),
                                          transforms.RandomHorizontalFlip(),
                                          transforms.RandomApply([color_jitter], p=0.8),
                                          transforms.RandomGrayscale(p=0.2),
                                          GaussianBlur(kernel_size=int(0.1 * input_shape[0])),
                                          transforms.ToTensor()])

    return data_transforms

class GaussianBlur(object):
    # Implements Gaussian blur as described in the SimCLR paper
    def __init__(self, kernel_size, min=0.1, max=2.0):
        self.min = min
        self.max = max
        # kernel size is set to be 10% of the image height/width
        self.kernel_size = kernel_size

    def __call__(self, sample):
        sample = np.array(sample)

        # blur the image with a 50% chance
        prob = np.random.random_sample()

        if prob < 0.5:
            sigma = (self.max - self.min) * np.random.random_sample() + self.min
            sample = cv2.GaussianBlur(sample, (self.kernel_size, self.kernel_size), sigma)

        return sample

max_stars_repo_name: gleb-t/SimCLR

max_issues_repo_name: gleb-t/SimCLR

max_forks_repo_name: gleb-t/SimCLR


text:
from pathlib import Path
import numpy as np
import h5py
import yaml
from tqdm import tqdm
import os
import sys

sys.path.append(os.getcwd())

try:
    from utils.dataset import read_info
except ImportError:
    exit(-1)

is_inside = 'INSIDE_DOCKER' in os.environ.keys() and \
            bool(os.environ['INSIDE_DOCKER'])

def write_samples(events, images, image_ts, img2event_map, out_dir, ts0):
    for i, (b, e, start_ts, stop_ts) in tqdm(enumerate(zip(img2event_map[:-1],
                                                           img2event_map[1:],
                                                           image_ts[:-1],
                                                           image_ts[1:])),
                                             total=img2event_map.size-1):

        # very strange fix, but it is required
        frame_events = np.array(events[b+1:e+1])
        assert frame_events[0, 2] >= start_ts, 'The first event is before ' \
                                               'the first image'
        assert b < 0 or events[b, 2] <= start_ts, 'Some events are missed'
        assert frame_events[-1, 2] <= stop_ts, \
               'The last event is after the second image'
        assert e+1 >= events.shape[0] or events[e+1, 2] >= stop_ts, \
               'Some events are missed'
        frame_events[:, 2] -= ts0
        image1 = np.array(images[i])
        image2 = np.array(images[i+1])
        with h5py.File(str(out_dir/f'{i:06d}.hdf5'), 'w') as of:
            of.create_dataset('image1', data=image1)
            of.create_dataset('image2', data=image2)
            of.create_dataset('events', data=frame_events)
            of.create_dataset('start', data=start_ts - ts0)
            of.create_dataset('stop', data=stop_ts - ts0)

script_dir = Path(__file__).resolve().parent.parent
if is_inside:
    data_dir = Path('/data')
    info_dir = data_dir/'info'
else:
    data_dir = (script_dir/'..'/'data').resolve()
    info_dir = script_dir/'data'/'info'

config_dir = script_dir/'config'
raw_data_dir = data_dir/'raw'
training_dir = data_dir/'training'

config_path = config_dir/'training_datasets.yml'
with open(config_path, 'r') as f:
    try:
        config = yaml.safe_load(f)
    except yaml.YAMLError as exc:
        print(exc)

for ds_name, ds_config in config.items():
    # dir with dataset data
    ds_raw_dir = raw_data_dir/ds_name
    # dir to write samples
    ds_training_dir = training_dir/ds_name
    # dir with dataset info: first timestamp for each sequence
    info_file = info_dir/(ds_name + '.hdf5')
    ds_info = read_info(str(info_file))
    for seq_name, seq_config in ds_config.items():
        # file with sequence data
        seq_raw_file = ds_raw_dir/seq_name[:-1]/(seq_name+'_data.hdf5')
        # dir to write sequence samples
        seq_training_dir = ds_training_dir/seq_name
        # begin and end of the training part
        start_ts = seq_config['start']
        stop_ts = seq_config['stop']
        # initial timestamp of the sequence
        t0 = ds_info[seq_name]

        if start_ts is None:
            start_ts = 0

        # sure output dir exists
        seq_training_dir.mkdir(parents=True, exist_ok=True)

        with h5py.File(str(seq_raw_file), 'r') as data:
            events = data['davis']['left']['events']
            image_ts = data['davis']['left']['image_raw_ts']
            images = data['davis']['left']['image_raw']
            img2event_map = np.array(data['davis']
                                         ['left']
                                         ['image_raw_event_inds'],
                                     dtype=np.int)

            # mask of images to use
            mask = image_ts >= t0 + start_ts
            if stop_ts is not None:
                mask = np.logical_and(mask, image_ts <= t0 + stop_ts)

            image_ts = image_ts[mask]
            images = images[mask, :]
            img2event_map = img2event_map[mask]

            write_samples(events, images, image_ts,
                          img2event_map, seq_training_dir, t0)


text:
import os
import torch
import torch.utils.data
import cv2
import h5py
import numpy as np
from util import Kernels
from torch import nn, optim
from torch.autograd import Variable
from torch.nn import functional as F
from torchvision import datasets, transforms
from torchvision.utils import save_image
from dataset import Dataset

# changed configuration to this instead of argparse for easier interaction
CUDA = True
SEED = 1234
BATCH_SIZE = 16
LOG_INTERVAL = 10
EPOCHS = 10
no_of_sample = 10
patch_size = 64

torch.manual_seed(SEED)
if CUDA:
    torch.cuda.manual_seed(SEED)

# DataLoader instances will load tensors directly into GPU memory
kwargs = {'num_workers': 4, 'pin_memory': True} if CUDA else {}

# Use full default dataset for training
dataset_train = Dataset(patch_size=patch_size, stride=32, size=6400, seed=SEED)
# shuffle data at every epoch
train_loader = torch.utils.data.DataLoader(
    dataset=dataset_train, batch_size=BATCH_SIZE, shuffle=True, **kwargs
)

dataset_test = Dataset(patch_size=patch_size, stride=32, size=640, seed=SEED)
# Same for test data
test_loader = torch.utils.data.DataLoader(
    dataset=dataset_test, batch_size=BATCH_SIZE, shuffle=True, **kwargs
)

class Conv_Block(nn.Module):
    def __init__(self, in_channels, out_channels, kernel_size, padding, stride, pool_kernel_size=(2, 2)):
        super(Conv_Block, self).__init__()
        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size, padding, stride)
        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size, padding, stride)
        self.pool = nn.MaxPool2d(pool_kernel_size)

    def forward(self, x):
        x = F.elu(self.conv1(x))
        x = F.elu(self.conv2(x))
        x = self.pool(x)

        return x

class VAE(nn.Module):
    def __init__(self, patch_size):
        super(VAE, self).__init__()

        self.patch_size = patch_size

        # Encoder
        self.block1 = Conv_Block(1, 64, (3, 3), 1, 1)  # 32
        self.block2 = Conv_Block(64, 128, (3, 3), 1, 1)  # 16
        self.block3 = Conv_Block(128, 32, (3, 3), 1, 1)  # 8

        # Decoder
        self.fct_decode = nn.Sequential(
            nn.Conv2d(16, 64, (3, 3), padding=1),
            nn.ELU(),
            nn.Upsample(scale_factor=2, mode='nearest'),  # 16
            nn.Conv2d(64, 64, (3, 3), padding=1),
            nn.ELU(),
            nn.Upsample(scale_factor=2, mode='nearest'),  # 32
            nn.Conv2d(64, 16, (3, 3), padding=1),
            nn.ELU(),
            nn.Upsample(scale_factor=2, mode='nearest'),  # 64
        )

        self.final_decod_mean = nn.Conv2d(16, 1, (3, 3), padding=1)

    def encode(self, x: Variable) -> (Variable, Variable):
        x = x.view(-1, 1, self.patch_size, self.patch_size)
        x = F.elu(self.block1(x))
        x = F.elu(self.block2(x))
        x = F.elu(self.block3(x))

        return x[:, :16, :, :], x[:, 16:, :, :]  # output shape - batch_size x 16 x 8 x 8

    def reparameterize(self, mu: Variable, logvar: Variable) -> Variable:

        if self.training:
            # multiply log variance with 0.5, then in-place exponent
            # yielding the standard deviation

            sample_z = []
            for _ in range(no_of_sample):
                std = logvar.mul(0.5).exp_()  # type: Variable
                eps = Variable(std.data.new(std.size()).normal_())
                sample_z.append(eps.mul(std).add_(mu))

            return sample_z

        else:
            # During inference, we simply spit out the mean of the
            # learned distribution for the current input.  We could
            # use a random sample from the distribution, but mu of
            # course has the highest probability.
            return mu

    def decode(self, z: Variable) -> Variable:
        z = self.fct_decode(z)
        z = self.final_decod_mean(z)
        z = F.sigmoid(z)

        return z.view(-1, self.patch_size * self.patch_size)

    def forward(self, x: Variable) -> (Variable, Variable, Variable):
        mu, logvar = self.encode(x.view(-1, self.patch_size * self.patch_size))
        z = self.reparameterize(mu, logvar)
        if self.training:
            return [self.decode(z) for z in z], mu, logvar
        else:
            return self.decode(z), mu, logvar

    def loss_function(self, recon_x, x, mu, logvar) -> Variable:
        # how well do input x and output recon_x agree?

        if self.training:
            BCE = 0
            for recon_x_one in recon_x:
                BCE += F.binary_cross_entropy(recon_x_one, x.view(-1, self.patch_size * self.patch_size))
            BCE /= len(recon_x)
        else:
            BCE = F.binary_cross_entropy(recon_x, x.view(-1, self.patch_size * self.patch_size))

        # KLD is Kullback–Leibler divergence -- how much does one learned
        # distribution deviate from another, in this specific case the
        # learned distribution from the unit Gaussian

        # see Appendix B from VAE paper:
        # Kingma and Welling. Auto-Encoding Variational Bayes. ICLR, 2014
        # https://arxiv.org/abs/1312.6114
        # - D_{KL} = 0.5 * sum(1 + log(sigma^2) - mu^2 - sigma^2)
        # note the negative D_{KL} in appendix B of the paper
        KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())
        # Normalise by same number of elements as in reconstruction
        KLD /= BATCH_SIZE * self.patch_size * self.patch_size

        return BCE + KLD

model = VAE(patch_size)
if CUDA:
    model.cuda(0)

optimizer = optim.Adam(model.parameters(), lr=1e-3)

def train(epoch, train_loader=train_loader):
    # toggle model to train mode
    model.train()
    train_loss = 0

    for batch_idx, data in enumerate(train_loader):
        data = Variable(data[:,:,:,:,0])
        if CUDA:
            data = data.cuda(0)
        optimizer.zero_grad()

        # push whole batch of data through VAE.forward() to get recon_loss
        recon_batch, mu, logvar = model(data)
        # calculate scalar loss
        loss = model.loss_function(recon_batch, data, mu, logvar)
        # calculate the gradient of the loss w.r.t. the graph leaves
        # i.e. input variables -- by the power of pytorch!
        loss.backward()
        train_loss += loss.item()
        optimizer.step()
        if batch_idx % LOG_INTERVAL == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(epoch, batch_idx * len(data), len(train_loader.dataset),
                       100. * batch_idx / len(train_loader),
                       loss.item() / len(data)))

    print('====> Epoch: {} Average loss: {:.4f}'.format(epoch, train_loss / len(train_loader.dataset)))

def test(epoch):
    model.eval()
    test_loss = 0

    # each data is of BATCH_SIZE (default 16) samples
    for i, data in enumerate(test_loader):
        if CUDA:
            # make sure this lives on the GPU
            data = data.cuda(0)

        # we're only going to infer, so no autograd at all required: volatile=True
        data = Variable(data[:,:,:,:,0], volatile=True)
        recon_batch, mu, logvar = model(data)
        test_loss += model.loss_function(recon_batch, data, mu, logvar).item()
        if i == 0:
            n = min(data.size(0), 8)
            # for the first 128 batch of the epoch, show the first 8 input digits
            # with right below them the reconstructed output digits
            comparison = torch.cat([data[:n],
                                    recon_batch.view(BATCH_SIZE, 1, patch_size, patch_size)[:n]])
            save_image(comparison.data.cpu(),
                       './vae_generated/reconstruction_' + str(epoch) + '.png', nrow=n)

    test_loss /= len(test_loader.dataset)
    print('====> Test set loss: {:.4f}'.format(test_loss))

def augment_with_vae(patch_size=64, stride=32, size='full', seed=1234, gksize=11, gsigma=3):
    '''
    Create the augmented dataset and save it
    Inputs:
        patch_size: the patch_size of the dataset to augment
        stride: the stride of the dataset to augment
        size: the size of the dataset to use for vae training
        seed: the seed of the dataset to use for vae training
        gksize: the gaussian blur kernel size
        gsigma: the gaussian blur kernel sigma parameter
    '''

    print('agumenting dataset with vae')

    # Load the dataset to use for training
    ds = Dataset(patch_size=patch_size, stride=stride, size=size, seed=seed)
    loader = torch.utils.data.DataLoader(
        dataset=ds,
        batch_size=16,
        shuffle=True,
        **kwargs
    )

    # Train the VAE
    for epoch in range(1, EPOCHS + 1):
        train(epoch, train_loader=loader)

    # Augment and save the dataset
    augmented_file_name = 'datasets/train_ps%d_stride%d_vae.h5' % (patch_size, stride)
    h5f = h5py.File(augmented_file_name, 'w')

    # Create the gaussian blur kernel
    kernel = Kernels.kernel_2d(gksize, gsigma)

    loader = torch.utils.data.DataLoader(
        dataset=ds,
        batch_size=1,
        shuffle=False
    )

    for patch_num, patch in enumerate(loader):
        patch = patch[0].numpy()
        h5f.create_dataset(str(patch_num), data=patch)

    # To be fair with the standard data augmentation method, we create 7 times the number
    # of images in the train set to augment it
    for i in range(7 * len(loader)):
        sample = Variable(torch.randn(1, 16, 8, 8))
        if CUDA:
            sample = sample.cuda(0)
        sample = model.decode(sample).cpu().detach().numpy()
        new_patch = sample.reshape(1, patch_size, patch_size, 1)
        blurred = np.expand_dims(
            np.expand_dims(
                cv2.filter2D(new_patch[0], -1, kernel, borderType=cv2.BORDER_CONSTANT),
                -1
            ),
            0
        )
        data = np.concatenate((new_patch, blurred), axis=-1)
        h5f.create_dataset(str(len(loader) + i), data=data)

    h5f.close()
    print('dataset augmentation with vae done')

if __name__ == "__main__":

    for epoch in range(1, EPOCHS + 1):
        train(epoch)
        test(epoch)

        # 64 sets of random ZDIMS-float vectors, i.e. 64 locations
        sample = Variable(torch.randn(64, 16, 8, 8))
        if CUDA:
            sample = sample.cuda(0)
        sample = model.decode(sample).cpu()

        # save out as an 8x8 matrix of images
        # this will give you a visual idea of how well latent space can generate things
        save_image(sample.data.view(64, 1, patch_size, patch_size),'./vae_generated/reconstruction' + str(epoch) + '.png')

max_stars_repo_name: majedelhelou/PriorLearning

max_issues_repo_name: majedelhelou/PriorLearning

max_forks_repo_name: majedelhelou/PriorLearning


text:
# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/plotting.manifolds.ipynb (unless otherwise specified).

__all__ = ['plot_manifold_2d', 'plot_manifold_3d', 'plot_manifold_featurespace', 'plot_manifold_3d_set']

# Cell
import matplotlib.pyplot as plt
import matplotlib as mpl
mpl.rcParams['text.usetex'] = True
mpl.rcParams['text.latex.preamble'] = [r'\usepackage{amsmath}'] #for \text command
import numpy as np

# Cell
import numpy as np

def plot_manifold_2d(data, s, alpha, c, title, title_color = 'black'):
    fig, axs = plt.subplots(1,1, figsize = (20,15))

    axs.set_axis_off()
    s = 1
    alpha = .5
    selected_points = list(range(data.shape[0]))
    x = data[:, 0][selected_points]
    y = data[:, 1][selected_points]
    ax = fig.add_subplot(1,1,1)

    ax.scatter(x, y,  s=s, alpha=alpha, marker='.',vmin=0,vmax=np.pi, c=c )
    ax.set_xlabel(r'$\phi_1$', fontsize = 140)
    ax.set_ylabel(r'$\phi_2$', fontsize = 140)
    ax.set_title(title, fontsize = 200, color= title_color)

    xmin = np.float(np.format_float_positional(data[:, 0][selected_points].min(), precision=2, fractional=False))
    xmax = np.float(np.format_float_positional(data[:, 0][selected_points].max(), precision=2, fractional=False))
    ymin = np.float(np.format_float_positional(data[:, 1][selected_points].min(), precision=2, fractional=False))
    ymax = np.float(np.format_float_positional(data[:, 1][selected_points].max(), precision=2, fractional=False))
#     ax.set_xticks([xmin, xmax])
#     ax.set_yticks([ymin, ymax])
    ax.set_xticks([])
    ax.set_yticks([])
    ax.spines['right'].set_visible(False)
    ax.spines['top'].set_visible(False)
    #ax.tick_params(labelsize=30)

# Cell
def plot_manifold_3d(data, s, alpha, c, title, title_color = 'black'):

    fig, axs = plt.subplots(1,1, figsize = (15,10))
    axs.set_axis_off()

    selected_points = list(range(data.shape[0]))
    x = data[:, 0][selected_points]
    y = data[:, 1][selected_points]
    z = data[:, 2][selected_points]

    ax = fig.add_subplot(1,1,1, projection='3d')
    ax.scatter(x, y, z, s=s,c = c, alpha=alpha, marker='.',vmin=0,vmax=np.pi)
    ax.set_xlabel(r'$\phi_1$', fontsize = 90)
    ax.set_ylabel(r'$\phi_2$', fontsize = 90)
    ax.set_zlabel(r'$\phi_3$', fontsize = 90)
    ax.set_title(title, fontsize = 150, color = title_color)

    xmin = np.float(np.format_float_positional(data[:, 0][selected_points].min(), precision=2, fractional=False))
    xmax = np.float(np.format_float_positional(data[:, 0][selected_points].max(), precision=2, fractional=False))
    ymin = np.float(np.format_float_positional(data[:, 1][selected_points].min(), precision=2, fractional=False))
    ymax = np.float(np.format_float_positional(data[:, 1][selected_points].max(), precision=2, fractional=False))
    zmin = np.float(np.format_float_positional(data[:, 2][selected_points].min(), precision=2, fractional=False))
    zmax = np.float(np.format_float_positional(data[:, 2][selected_points].max(), precision=2, fractional=False))

    #ax.set_xticks([xmin, xmax])
    #ax.set_yticks([ymin, ymax])
    #ax.set_zticks([zmin, zmax])
    ax.set_xticks([])
    ax.set_yticks([])
    ax.set_zticks([])
    #ax.tick_params(labelsize=30)
    ax.xaxis.set_pane_color((1.0, 1.0, 1.0, 0.0))
    ax.yaxis.set_pane_color((1.0, 1.0, 1.0, 0.0))
    ax.zaxis.set_pane_color((1.0, 1.0, 1.0, 0.0))
    # make the grid lines transparent
    ax.xaxis._axinfo["grid"]['color'] =  (1,1,1,0)
    ax.yaxis._axinfo["grid"]['color'] =  (1,1,1,0)
    ax.zaxis._axinfo["grid"]['color'] =  (1,1,1,0)

# Cell
import math
import matplotlib.pyplot as plt
import numpy as np

def plot_manifold_featurespace(data,title,ncord = 6):

    s = 100
    alpha = .5
    fig, axes = plt.subplots(ncord,ncord, figsize = (40,40))
    xmins = np.zeros(ncord)
    xmaxs = np.zeros(ncord)
    xmeans = np.zeros(ncord)
    for d in range(ncord):
        xmins[d] = data[:,d].min()
        xmaxs[d] = data[:,d].max()
        xmeans[d] = (xmins[d] +  xmaxs[d]) / 2

    gap = (xmaxs - xmins).max()

    for d in range(ncord):
        xmins[d] = xmeans[d] - gap / 2
        xmaxs[d] = xmeans[d] + gap / 2

    for d in range(ncord):
        print(d)
        for e in range(ncord):
            xticks = [math.ceil(xmins[d]), 0 ,math.ceil(xmaxs[d])]
            yticks = [math.ceil(xmins[e]), 0 ,math.ceil(xmaxs[e])]
            if d != e:
                axes[e,d].scatter(data[:,d],data[:,e], s = .1, alpha = .1)
                axes[e,d].set_xlim(xmins[d], xmaxs[d])
                axes[e,d].set_ylim(xmins[e], xmaxs[e])

                if d == 0:
                    #axes[e,d].set_xticks(xticks)
                    axes[e,d].set_yticks(yticks)
                else:
                    axes[e,d].set_yticks([])
                if e == (ncord-1):
                    axes[e,d].set_xticks(xticks)
                else:
                    axes[e,d].set_xticks([])
                axes[e,d].tick_params(labelsize=60)
            if d == e:

                axes[d,e].hist(data[:,d])
                axes[e,d].tick_params(labelsize=50)
                axes[e,d].tick_params(labelsize=50)
                if e == (ncord-1):
                    axes[e,d].set_xticks(xticks)
                else:
                    axes[e,d].set_xticks([])

    for d in range(ncord):
        axes[ncord- 1,d].set_xlabel(r'$\xi_{{{}}}$'.format(d+1), fontsize= 110)
        axes[d,0].set_ylabel(r'$\xi_{{{}}}$'.format(d+1), fontsize= 110)

    plt.tight_layout()
    fig = plt.gcf()
    fig.subplots_adjust(top=0.9)
    fig.suptitle(title,y=.95,x = .53, fontsize=120)

# Cell

def plot_manifold_3d_set(data, s, alpha, gb, titles,sub, title_colors, title):

    nb = len(sub)

    fig, axs = plt.subplots(math.floor(nb / 4) + 1,4, figsize = (40, 10 *( math.floor(nb / 4) + 1),))

    for r in range((math.floor(nb / 4)+1 )* 4):
        axs[math.floor(r / 4), r %4 ].set_axis_off()

    selected_points = list(range(data.shape[0]))
    x = data[:, 0][selected_points]
    y = data[:, 1][selected_points]
    z = data[:, 2][selected_points]
    for r in range(nb):
        print(r)
        ax = fig.add_subplot(math.floor(nb / 4)+1, 4 ,r+1, projection='3d')
        ax.scatter(x, y, z, c = gb[:,r], s=s, alpha=alpha, marker='.',vmin=0,vmax=np.pi)
        ax.set_xlabel(r'$\phi_1$', fontsize = 60)
        ax.set_ylabel(r'$\phi_2$', fontsize = 60)
        ax.set_zlabel(r'$\phi_3$', fontsize = 60)
        ax.set_title(r'$g_{{{}}}$'.format(sub[r]) + ' ' + str(titles[sub[r]]+1), fontsize = 60, color = title_colors[r])
        #ax.set_axis_off()
        xmin = np.float(np.format_float_positional(data[:, 0][selected_points].min(), precision=2, fractional=False))
        xmax = np.float(np.format_float_positional(data[:, 0][selected_points].max(), precision=2, fractional=False))
        ymin = np.float(np.format_float_positional(data[:, 1][selected_points].min(), precision=2, fractional=False))
        ymax = np.float(np.format_float_positional(data[:, 1][selected_points].max(), precision=2, fractional=False))
        zmin = np.float(np.format_float_positional(data[:, 2][selected_points].min(), precision=2, fractional=False))
        zmax = np.float(np.format_float_positional(data[:, 2][selected_points].max(), precision=2, fractional=False))
        ax.set_xticks([])
        ax.set_yticks([])
        ax.set_zticks([])
        ax.set_xticklabels([])
        ax.set_yticklabels([])
        ax.set_zticklabels([])
        ax.tick_params(labelsize=40)

        ax.xaxis.set_pane_color((1.0, 1.0, 1.0, 0.0))
        ax.yaxis.set_pane_color((1.0, 1.0, 1.0, 0.0))
        ax.zaxis.set_pane_color((1.0, 1.0, 1.0, 0.0))
        # make the grid lines transparent
        ax.xaxis._axinfo["grid"]['color'] =  (1,1,1,0)
        ax.yaxis._axinfo["grid"]['color'] =  (1,1,1,0)
        ax.zaxis._axinfo["grid"]['color'] =  (1,1,1,0)

    #plt.tight_layout()
    fig = plt.gcf()
    fig.subplots_adjust(top=0.9)
    fig.suptitle(title,y=.95,x = .53, fontsize=80)

max_stars_repo_name: sjkoelle/montlake

max_issues_repo_name: sjkoelle/montlake

max_forks_repo_name: sjkoelle/montlake


text:
from Dirs import *
import Layers
import Model
from Data import *
import pandas as pd
import numpy as np
import time

def Lat_Gen():
    print("Iniciando...")
    start = time.time()

    model_list = ["CAE","CVAE"]
    #std_list = [1.0,0.5, 0.1, 1e-13]
    std_list = [1.0,0.5, 0.1, 1e-13]
    lat_dim_list = [2,3,4,5,6]

    for i in range(len(model_list)):
        for j in range(len(std_list)):
            for k in range(len(lat_dim_list)):

                model = model_list[i]
                lat_dim = lat_dim_list[k]
                std = std_list[j]

                if (model == "AE" or model == "CAE") and j != 0:
                    continue

                else:

                    #print(f"[CAE]Lat_Dim={lat_dim}")

                    #model = Model.VAE.build(dim=lat_dim, std=std)
                    #model.load_weights(models_dir + "/" + f"[VAE]Lat_Dim={lat_dim} | STD={std}.hdf5")
                    #modelo = Layers.VAE.build(dim=lat_dim ,std= std)

                    #Model.AAE.build(dim = lat_dim, std= std, mode="Test")
                    #predict = modelo.predict(data_normalized)
                    #df = pd.DataFrame(data = predict)
                    #df.to_csv(latentSpace_dir + "/" + f"[AAE]Lat_Dim={lat_dim} | STD={std}.csv", index=False)
                    #break
                    if model == "CVAE":
                        print(f"[CVAE]Lat_Dim={lat_dim} | STD={std}")
                        model = Model.CVAE.build(dim=lat_dim, std=std)
                        model.load_weights(models_dir + "/" + f"[CVAE]Lat_Dim={lat_dim} | STD={std}.hdf5")
                        modelo = Layers.CVAE.build(dim=lat_dim ,std= std)
                        modelo.set_weights(model.get_weights())
                        predict = modelo.predict(cx_test, batch_size = 16)
                        df = pd.DataFrame(data = predict)
                        df.to_csv(latentSpace_dir + "/" + f"[CVAE]Lat_Dim={lat_dim} | STD={std}", index=False)

                    if model == "VAE":
                        print(f"[VAE]Lat_Dim={lat_dim} | STD={std}")
                        model = Model.VAE.build(dim=lat_dim, std=std)
                        model.load_weights(models_dir + "/" + f"[VAE]Lat_Dim={lat_dim} | STD={std}.hdf5")
                        modelo = Layers.VAE.build(dim=lat_dim ,std= std)
                        modelo.set_weights(model.get_weights())
                        predict = modelo.predict(x_test, batch_size = 16)
                        df = pd.DataFrame(data = predict)
                        df.to_csv(latentSpace_dir + "/" + f"[VAE]Lat_Dim={lat_dim} | STD={std}", index=False)

                    if model == "AE":
                        print(f"[AE]Lat_Dim={lat_dim}")
                        model = Model.AE.build(dim=lat_dim)
                        model.load_weights(models_dir + "/" + f"[AE]Lat_Dim={lat_dim}.hdf5")
                        modelo = Layers.AE.build(dim=lat_dim)
                        modelo.set_weights(model.get_weights())
                        predict = modelo.predict(x_test, batch_size = 16)
                        #print(modelo.summary())
                        df = pd.DataFrame(data = predict)
                        df.to_csv(latentSpace_dir + "/" + f"[AE]Lat_Dim={lat_dim}", index=False)

                    if model == "CAE":
                        print(f"[CAE]Lat_Dim={lat_dim}")
                        model = Model.CAE.build(dim=lat_dim)
                        #print(model.get_weights()[0][0])
                        model.load_weights(models_dir + "/" + f"[CAE]Lat_Dim={lat_dim}.hdf5")
                        #print(model.get_weights()[0][0])
                        modelo = Layers.CAE.build(dim=lat_dim)
                        #print(modelo.get_weights()[0][0])
                        modelo.set_weights(model.get_weights())
                        #print(modelo.get_weights()[0][0])
                        predict = modelo.predict(cx_test, batch_size = 200)
                        #print(modelo.summary())
                        df = pd.DataFrame(data = predict)
                        df.to_csv(latentSpace_dir + "/" + f"[CAE]Lat_Dim={lat_dim}", index=False)

    finish = time.time()
    tempo = finish - start
    print("Minutos: {}".format(int(tempo/60)))
Lat_Gen()

max_stars_repo_name: patrick-schubert/emission-line-galaxy

max_issues_repo_name: patrick-schubert/emission-line-galaxy

max_forks_repo_name: patrick-schubert/emission-line-galaxy


text:
import cv2
import numpy as np
import os

labels = {0: "angle-boot", 1: "bag", 2: "coat", 3: "dress", 4: "pullover",
          5: "sandal", 6: "shirt", 7: "sneaker", 8: "t-shirt", 9: "trouser"}

class FashionClassifier:

    def __init__(self, model_path, backend = 3, scalefactor=1.0, size=(), mean=(), swapRB=False, crop=False, ddepth=cv2.CV_32F):

        if os.path.isdir(model_path):
            self.net = cv2.dnn.readNetFromModelOptimizer(model_path+"/model.xml", model_path+"/model.bin")
        else:
            self.net = cv2.dnn.readNetFromONNX(model_path)

        self.net.setPreferableBackend(backend)
        self.net.setPreferableTarget(0)

        self.scalefactor = scalefactor
        self.size = size
        self.mean = mean
        self.swapRB = swapRB
        self.crop = crop
        self.ddepth = ddepth

    # Convert the raw result to the string labels
    def process_output(self, out):
        out = out.flatten()
        classId = np.argmax(out)
        return labels[classId]

    def predict(self, img):
        # Preprocess the image to the network needs
        blob = cv2.dnn.blobFromImage(
            img, self.scalefactor, self.size, self.mean, self.swapRB, self.crop, self.ddepth)
        self.net.setInput(blob)
        out = self.net.forward()
        return self.process_output(out)


text:
from ..core.project_manager import ProjectManager
from uquake.core.stream import Stream
from uquake.core.logging import logger
from pathlib import Path
import os
import shutil
import numpy as np
from time import time
from uquake.core.util import tools
from xseis2 import xspy

class Interloc(ProjectManager):
    def __init__(self, base_projects_path, project_name, network_code,
                 **kwargs):

        super().__init__(base_projects_path, project_name, network_code,
                         **kwargs)

        self.files.magnitude_settings = self.paths.config / \
                                        'interloc_settings.toml'

        if not self.files.magnitude_settings.is_file():
            settings_template = Path(os.path.realpath(__file__)).parent / \
                                '../settings/interloc_settings_template.toml'

            shutil.copyfile(settings_template,
                            self.files.magnitude_settings)

            super().__init__(base_projects_path, project_name, network_code,
                             **kwargs)

    def locate(self, stream: Stream):

        logger.info("pipeline: interloc")

        # TODO: copy not necessary test application is broken

        nthreads = self.settings.interloc.nthreads
        fixed_wlen_sec = self.settings.interloc.fixed_wlen_sec
        sample_rate_decimated = self.settings.interloc.samplerate_decimated
        pair_dist_min = self.settings.interloc.pair_dist_min
        pair_dist_max = self.settings.interloc.pair_dist_max
        cc_smooth_length_sec = self.settings.interloc.cc_smooth_length_sec

        whiten_corner_freqs = np.array(
            self.settings.interloc.whiten_corner_freqs, dtype=np.float32)

        stalocs = self.htt.locations
        ttable = (self.htt.hf["ttp"][:] * sample_rate_decimated).astype(
            np.uint16)
        ttable_s = (self.htt.hf["tts"][:] * sample_rate_decimated).astype(
            np.uint16)
        ngrid = ttable.shape[1]
        ttable_row_ptrs = np.array(
            [row.__array_interface__["data"][0] for row in ttable])
        ttable_row_ptrs_s = np.array(
            [row.__array_interface__["data"][0] for row in ttable_s])

        logger.info("preparing data for Interloc")
        t4 = time()

        # remove channels which do not have matching ttable entries
        # This should be handled upstream

        for trace in stream:
            station = trace.stats.station
            component = trace.stats.channel
            if trace.stats.site not in self.htt.sites:
                logger.warning(f'Sensor {station} not in the H5 travel time '
                               f'file sensor list... removing trace for '
                               f'{station} and component {component}')
                stream.remove(trace)
            elif np.max(trace.data) == 0:
                # from ipdb import set_trace; set_trace()
                logger.warning(f'trace for component {component} of sensor'
                               f' {station} contains only zero... removing '
                               f'trace')
                stream.remove(trace)
            elif trace.stats.site in self.settings.sites.black_list:
                logger.warning(f'sensor {station} is in the black list... '
                               f'removing trace for sensor {station} and '
                               f'component {component}')

                stream.remove(trace)

        sample_rate = stream[0].stats.sampling_rate
        decimate_factor = int(sample_rate / sample_rate_decimated)
        if decimate_factor == 0:
            stream = stream.resample(sample_rate_decimated)
            decimate_factor = 1

        data, sample_rate, t0 = stream.as_array(fixed_wlen_sec)
        data = np.nan_to_num(data)
        decimate_factor = int(sample_rate / sample_rate_decimated)
        if decimate_factor > 1:
            data = tools.decimate(data, sample_rate, decimate_factor)
        channel_map = stream.channel_map().astype(np.uint16)

        ikeep = self.htt.index_sites(stream.unique_sites)
        debug_file = self.paths.debug / f'iloc_{str(t0)}.npz'
        t5 = time()
        logger.info(
            "done preparing data for Interloc in %0.3f seconds" % (t5 - t4))

        debug_level = self.settings.interloc.debug_level
        debug_level = 0

        logger.info("Locating event with Interloc")
        t6 = time()
        logger.info(
            "sample_rate_decimated {}, ngrid {}, nthreads {}, debug {}, "
            "debug_file {}",
            sample_rate_decimated, ngrid, nthreads,
            debug_level,
            debug_file)

        from ipdb import set_trace
        set_trace()
        out = xspy.pySearchOnePhase(
            data,
            int(sample_rate_decimated),
            channel_map,
            stalocs[ikeep],
            ttable_row_ptrs[ikeep],
            ngrid,
            whiten_corner_freqs,
            pair_dist_min,
            pair_dist_max,
            cc_smooth_length_sec,
            nthreads,
            debug_level,
            str(debug_file)
        )

        out_s = xspy.pySearchOnePhase(
            data,
            int(sample_rate_decimated),
            channel_map,
            stalocs[ikeep],
            ttable_row_ptrs_s[ikeep],
            ngrid,
            whiten_corner_freqs,
            pair_dist_min,
            pair_dist_max,
            cc_smooth_length_sec,
            nthreads,
            debug_level,
            str(debug_file)
        )

        vmax, imax, iot = out
        vmax_s, imax_s, iot_s = out_s

        if vmax_s > vmax:
            vmax = vmax_s
            imax = imax_s
            iot = iot_s
            logger.info('stacking along the s-wave moveout curve yielded '
                        'better result')

        normed_vmax = vmax * fixed_wlen_sec
        lmax = self.htt.icol_to_xyz(imax)
        t7 = time()
        logger.info("Done locating event with Interloc in %0.3f seconds" % (
                t7 - t6))

        t0_epoch = t0.timestamp
        ot_epoch = t0_epoch + iot / sample_rate_decimated

        method = "%s" % ("INTERLOC",)

        logger.info("power: %.3f, ix_grid: %d, ix_ot: %d" % (vmax, imax, iot))
        logger.info("utm_loc: {}", lmax)
        logger.info("=======================================\n")
        logger.info("VMAX over threshold (%.3f)" % (vmax))

        self.response = {'x': lmax[0],
                         'y': lmax[1],
                         'z': lmax[2],
                         'vmax': vmax,
                         'normed_vmax': normed_vmax,
                         'event_time': ot_epoch,
                         'method': method}

        return self.response

max_stars_repo_name: jeanphilippemercier/uquake-useis

max_issues_repo_name: jeanphilippemercier/uquake-useis

max_forks_repo_name: jeanphilippemercier/uquake-useis


text:
from typing import List, Optional, Union, Callable, Dict, Any, Sequence

import numpy as np
from sklearn.base import ClassifierMixin, TransformerMixin
from sklearn.pipeline import Pipeline
from sklearn.model_selection import (
    BaseCrossValidator,
    BaseShuffleSplit,
    KFold,
    StratifiedKFold,
)
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import LinearSVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import (
    RandomForestClassifier,
    HistGradientBoostingClassifier,
)
from xgboost import XGBClassifier
from sklearn.dummy import DummyClassifier
from sklearn.utils.multiclass import type_of_target

from poniard.estimators.core import PoniardBaseEstimator
from poniard.plot.plot_factory import PoniardPlotFactory

class PoniardClassifier(PoniardBaseEstimator):
    """Cross validate multiple classifiers, rank them, fine tune them and ensemble them.

    PoniardClassifier takes a list/dict of scikit-learn estimators and compares their performance
    on a list/dict of scikit-learn metrics using a predefined scikit-learn cross-validation
    strategy.

    Parameters
    ----------
    estimators :
        Estimators to evaluate.
    metrics :
        Metrics to compute for each estimator. This is more restrictive than sklearn's scoring
        parameter, as it does not allow callable scorers. Single strings are cast to lists
        automatically.
    preprocess : bool, optional
        If True, impute missing values, standard scale numeric data and one-hot or ordinal
        encode categorical data.
    scaler :
        Numeric scaler method. Either "standard", "minmax", "robust" or scikit-learn Transformer.
    numeric_imputer :
        Imputation method. Either "simple", "iterative" or scikit-learn Transformer.
    custom_preprocessor :
        Preprocessor used instead of the default preprocessing pipeline. It must be able to be
        included directly in a scikit-learn Pipeline.
    numeric_threshold :
        Features with unique values above a certain threshold will be treated as numeric. If
        float, the threshold is `numeric_threshold * samples`.
    cardinality_threshold :
        Non-numeric features with cardinality above a certain threshold will be treated as
        ordinal encoded instead of one-hot encoded. If float, the threshold is
        `cardinality_threshold * samples`.
    cv :
        Cross validation strategy. Either an integer, a scikit-learn cross validation object,
        or an iterable.
    verbose :
        Verbosity level. Propagated to every scikit-learn function and estimator.
    random_state :
        RNG. Propagated to every scikit-learn function and estimator. The default None sets
        random_state to 0 so that cross_validate results are comparable.
    n_jobs :
        Controls parallel processing. -1 uses all cores. Propagated to every scikit-learn
        function and estimator.
    plugins :
        Plugin instances that run in set moments of setup, fit and plotting.
    plot_options :
        :class:poniard.plot.plot_factory.PoniardPlotFactory instance specifying Plotly format
        options or None, which sets the default factory.
    cache_transformations :
        Whether to cache transformations and set the `memory` parameter for Pipelines. This can
        speed up slow transformations as they are not recalculated for each estimator.

    Attributes
    ----------
    estimators_ :
        Estimators used for scoring.
    preprocessor_ :
        Pipeline that preprocesses the data.
    metrics_ :
        Metrics used for scoring estimators during fit and hyperparameter optimization.
    cv_ :
        Cross validation strategy.
    """

    def __init__(
        self,
        estimators: Optional[
            Union[Dict[str, ClassifierMixin], List[ClassifierMixin]]
        ] = None,
        metrics: Optional[Union[str, Dict[str, Callable], Sequence[str]]] = None,
        preprocess: bool = True,
        scaler: Optional[Union[str, TransformerMixin]] = None,
        numeric_imputer: Optional[Union[str, TransformerMixin]] = None,
        custom_preprocessor: Union[None, Pipeline, TransformerMixin] = None,
        numeric_threshold: Union[int, float] = 0.1,
        cardinality_threshold: Union[int, float] = 20,
        cv: Union[int, BaseCrossValidator, BaseShuffleSplit, Sequence] = None,
        verbose: int = 0,
        random_state: Optional[int] = None,
        n_jobs: Optional[int] = None,
        plugins: Optional[List[Any]] = None,
        plot_options: Optional[PoniardPlotFactory] = None,
        cache_transformations: bool = False,
    ):
        super().__init__(
            estimators=estimators,
            metrics=metrics,
            preprocess=preprocess,
            scaler=scaler,
            numeric_imputer=numeric_imputer,
            numeric_threshold=numeric_threshold,
            custom_preprocessor=custom_preprocessor,
            cardinality_threshold=cardinality_threshold,
            cv=cv,
            verbose=verbose,
            random_state=random_state,
            n_jobs=n_jobs,
            plugins=plugins,
            plot_options=plot_options,
            cache_transformations=cache_transformations,
        )

    @property
    def _base_estimators(self) -> List[ClassifierMixin]:
        return [
            LogisticRegression(
                random_state=self.random_state, verbose=self.verbose, max_iter=5000
            ),
            GaussianNB(),
            LinearSVC(
                random_state=self.random_state, verbose=self.verbose, max_iter=5000
            ),
            KNeighborsClassifier(),
            DecisionTreeClassifier(random_state=self.random_state),
            RandomForestClassifier(
                random_state=self.random_state, verbose=self.verbose, n_jobs=self.n_jobs
            ),
            HistGradientBoostingClassifier(
                random_state=self.random_state, verbose=self.verbose
            ),
            XGBClassifier(
                random_state=self.random_state,
                use_label_encoder=False,
            ),
            DummyClassifier(strategy="prior"),
        ]

    def _build_metrics(self) -> Union[Dict[str, Callable], List[str], Callable]:
        y = self.y
        if y.ndim > 1:
            return [
                "accuracy",
                "precision_macro",
                "recall_macro",
                "f1_macro",
            ]
        elif y.ndim == 1 and len(np.unique(y)) > 2:
            return [
                "roc_auc_ovr",
                "accuracy",
                "precision_macro",
                "recall_macro",
                "f1_macro",
            ]

        else:
            return [
                "roc_auc",
                "accuracy",
                "precision",
                "recall",
                "f1",
            ]

    def _build_cv(self) -> BaseCrossValidator:
        cv = self.cv or 5
        if isinstance(cv, int):
            if (self.y is not None) and (
                type_of_target(self.y) in ("binary", "multiclass")
            ):
                return StratifiedKFold(
                    n_splits=cv, shuffle=True, random_state=self.random_state
                )
            else:
                return KFold(n_splits=cv, shuffle=True, random_state=self.random_state)
        else:
            self._pass_instance_attrs(cv)
            return cv

max_stars_repo_name: rxavier/poniard

max_issues_repo_name: rxavier/poniard

max_forks_repo_name: rxavier/poniard


text:
import unittest
import numpy as np
import networkx as nx
from variationaltoolkit.objectives import modularity_obj

class TestObjectives(unittest.TestCase):

    def test_modularity_obj(self):
        w = np.array([[0,1,1,0,0,0],[1,0,1,0,0,0],[1,1,0,1,0,0],[0,0,1,0,1,1],[0,0,0,1,0,1],[0,0,0,1,1,0]])
        G = nx.from_numpy_matrix(w)
        B = nx.modularity_matrix(G, nodelist = list(range(6)))
        m = G.number_of_edges()

        x = np.array([0,0,0,1,1,1])
        N = 1
        y = np.array([0,0,0,1,1,0,1,1,1,1,1,1])
        M = 2

        self.assertTrue(abs(modularity_obj(x, N, B, m) + 10/28) < 1e-5)
        self.assertTrue(abs(modularity_obj(y, M, B, m) + 9/98) < 1e-5)

if __name__ == '__main__':
    unittest.main()

reason: import numpy,import networkx


text:
""" Thanks @nathancy on StackOverflow
    Website:
    stackoverflow.com/questions/56486710/animate-pyqtgraph-in-class?rq=1
"""
from pyqtgraph.Qt import QtCore, QtGui
from threading import Thread
import pyqtgraph as pg
import numpy as np
import random
import sys
import time
import os

import Simulation
import Layers
import Movement
import Space

class Environment(QtGui.QWidget):
    def __init__(self, sim, freq=0.01, change_space_at_step=10000):
        super().__init__()

        # the simulation module
        self.sim = sim

        # Desired Frequency (Hz) = 1 / self.FREQUENCY
        # USE FOR TIME.SLEEP (s)
        self.FREQUENCY = freq

        # Frequency to update plot (ms)
        # USE FOR TIMER.TIMER (ms)
        self.TIMER_FREQUENCY = self.FREQUENCY * 1000

        # Outer grid layout
        self.layout = QtGui.QGridLayout()

        self._make_env()
        self._make_data_plots()

        # Count the steps of the BV
        self.step_counter = 0

        # when to change the space
        self.time_to_change = change_space_at_step

        # start the simulation
        self.read_pos_thread()
        self.start()

    def _make_env(self):
        # Create Plot Widget
        self.env_widget = pg.PlotWidget()

        # Enable/disable plot squeeze (Fixed axis movement)
        # self.env_widget.plotItem.setMouseEnabled(x=False, y=False)
        max_pos = self.sim.get_max_pos()
        self.env_widget.setXRange(0, max_pos)
        self.env_widget.setYRange(0, max_pos)
        self.env_widget.setTitle('Environment')

        # Making moving BV
        self.moving_BV = pg.ScatterPlotItem()
        self.moving_BV.setBrush(255,0,0)
        # Add moving BV to the plot widget
        self.env_widget.addItem(self.moving_BV)

        # Making static stimuli
        self.stims = pg.ScatterPlotItem()
        self.stims.setData(pos=self.sim.get_stim())
        # Add stimuli to the plot widget
        self.env_widget.addItem(self.stims)

        # Add to the outer layoit
        self.layout.addWidget(self.env_widget, 0, 0, 2, 2)

        # set the scale of the first column (2 unit column width)
        self.layout.setColumnStretch(0, 2)

    def _make_data_plots(self):
        # the preference data
        self.pref_data = np.zeros(50, dtype=np.float64)

        # Making preference widget and plot
        self.pref_widget = pg.PlotWidget()
        self.pref_widget.setYRange(-100, 100)
        self.pref_widget.setTitle('Preference')
        self.pref_widget.addLegend(offset=(0, 0))

        self.pref_plot = pg.PlotCurveItem(name='actual')
        self.pref_widget.addItem(self.pref_plot)

        self.ideal_pref_plot = pg.PlotCurveItem(name='ideal')
        self.ideal_pref_plot.setPen(255, 255, 0)
        self.pref_widget.addItem(self.ideal_pref_plot)

        self.layout.addWidget(self.pref_widget, 0, 2, 1, 1)

        # the stimulus received
        o, g = self.sim.get_num_receptors()
        self.stim = np.zeros(o + g)
        self.olf_processed = np.zeros(o)
        self.recalled = np.zeros(g)
        self.gus_x_axis = np.arange(o, o+g)

        # Making stimulus widget and plot
        xticks = [(i, 'O{}'.format(i)) for i in range(o)] + \
                [(i+o, 'G{}'.format(i)) for i in range(g)]
        ax = pg.AxisItem(orientation='bottom')
        ax.setTicks([xticks])
        self.stim_widget = pg.PlotWidget(axisItems={'bottom': ax})
        self.stim_widget.setYRange(0, 3)
        self.stim_widget.setTitle('Stimulus')
        self.stim_widget.addLegend(offset=(0, 10))

        # received taste
        self.stim_plot = pg.PlotCurveItem(name='received')
        self.stim_widget.addItem(self.stim_plot)

        # recalled taste
        self.recalled_plot = pg.PlotCurveItem(name='recalled')
        self.recalled_plot.setPen(255, 0, 0)
        self.stim_widget.addItem(self.recalled_plot)

        # ideal taste
        self.ideal_gus_plot = pg.PlotCurveItem(name='ideal')
        self.ideal_gus_plot.setPen(255, 255, 0)
        self.stim_widget.addItem(self.ideal_gus_plot)

        self.layout.addWidget(self.stim_widget, 1, 2, 1, 1)

        # set the scale of the 3rd column
        self.layout.setColumnStretch(2, 1)

        # the processed olfactory information
        self.olf_processed_widget = pg.PlotWidget()
        self.olf_processed_widget.setTitle('Output of Olfactory Bulb')
        self.olf_processed_widget.setYRange(0, 1)
        self.olf_processed_plot = pg.PlotCurveItem()
        self.olf_processed_widget.addItem(self.olf_processed_plot)
        self.layout.addWidget(self.olf_processed_widget, 0, 3, 1, 1)

        # the associative memory weight matrix
        self.weight_widget = pg.PlotWidget()
        self.weight_widget.setTitle('Associative Memory Weight Matrix')
        self.weight_heat = pg.ImageItem()
        self.weight_widget.addItem(self.weight_heat)
        self.layout.addWidget(self.weight_widget, 1, 3, 1, 1)
        self.W = np.asarray(self.sim.get_asso_weight())

        # set the scale of the 4th column
        self.layout.setColumnStretch(3, 1)

    # Update plot
    def start(self):
        self.pos_update_timer = QtCore.QTimer()
        self.pos_update_timer.timeout.connect(self.plot_updater)
        self.pos_update_timer.start(self.TIMER_FREQUENCY)

    # Read in data using a thread
    def read_pos_thread(self):
        self.BV_pos = self.sim.get_BV_pos()
        self.pos_update_thread = Thread(target=self.read_pos, args=())
        self.pos_update_thread.daemon = True
        self.pos_update_thread.start()

    def read_pos(self):
        frequency = self.FREQUENCY
        while True:
            if self.step_counter == self.time_to_change:
                self.sim.change_space()
                self.step_counter = 0
                self.stims.setData(pos=self.sim.get_stim())

            if self.step_counter % 1000 == 0:
                self.sim.set_target()
            self.sim.step()
            self.step_counter += 1

            # update BV pos
            self.BV_pos = self.sim.get_BV_pos()

            # update pref
            self.pref_data = np.asarray(self.sim.get_pref())

            # update ideal pref
            self.ideal_pref_data = np.asarray(self.sim.get_ideal_pref())

            # update stimulus
            self.stim = np.asarray(self.sim.get_cur_stim())

            # update processed olfactory information
            self.olf_processed = np.asarray(self.sim.get_olf_bulb_output())

            # update recalled taste
            self.recalled = np.asarray(self.sim.get_recalled())

            # update ideal taste
            self.ideal_gus = np.asarray(self.sim.get_ideal_gus())
            if np.isnan(self.ideal_gus).any():
                print(np.asarray(self.BV_pos))

            # update weight
            self.W = np.asarray(self.sim.get_asso_weight())

            # sleep
            time.sleep(frequency)

    def plot_updater(self):
        self.moving_BV.setData(pos=self.BV_pos)
        self.pref_plot.setData(y=self.pref_data)
        self.ideal_pref_plot.setData(y=self.ideal_pref_data)
        self.stim_plot.setData(y=self.stim)
        self.olf_processed_plot.setData(y=self.olf_processed)
        self.recalled_plot.setData(x=self.gus_x_axis, y=self.recalled)
        self.ideal_gus_plot.setData(x=self.gus_x_axis, y=self.ideal_gus)
        self.weight_heat.setImage(self.W)

    def get_layout(self):
        return self.layout

    def get_BV_pos(self):
        return self.BV_pos

    def get_env_widget(self):
        return self.env_widget

if __name__ == '__main__':
    o = 10
    g = 5

    def mapping(x, y):
        for i in range(g):
            y[i] = 0

        for i in range(o):
            y[i // 2] += x[i]

        norm = np.linalg.norm(y)

        if norm > 1e-15:
            for i in range(g):
                y[i] /= norm

    space = Space.Space(20, o, g, mapping)

    gus = Layers.Single(g, lambda x: x)

    olf = Layers.LiHopfield(o, tau=5)

    m = Movement.RadMotor(200)

    asso = Layers.BAM(o, g, adapting_rate=1e-3, depression_rate=1e-10, enable_dep=True)

    pfunc = np.array([0, 0, 1, 1], dtype=np.int32)

    sim = Simulation.OnlineSim(olf, asso, gus, m, space, pfunc, mapping)

    # save imgs
    if not os.path.exists('img'):
        os.makedirs('img')

    space.save_img(name_prefices=('img/odor_space', 'img/taste_space'))
    olf.save_img(name='img/olf.png')
    asso.save_img(name='img/asso.png')
    sim.save_network_outer_img(name='img/outer_struct.png')

    # Create main application window
    app = QtGui.QApplication([])
    app.setStyle(QtGui.QStyleFactory.create("Cleanlooks"))
    mw = QtGui.QMainWindow()
    mw.setWindowTitle('Plot Example')

    # Create scrolling plot
    env_widget = Environment(sim)

    # Create and set widget layout
    # Main widget container
    cw = QtGui.QWidget()
    ml = QtGui.QGridLayout()
    cw.setLayout(ml)
    mw.setCentralWidget(cw)

    # Can use either to add plot to main layout
    #ml.addWidget(env_widget.get_env_widget(),0,0)
    ml.addLayout(env_widget.get_layout(),0,0)
    mw.show()

    # Start Qt event loop unless running in interactive mode or using pyside
    if (sys.flags.interactive != 1) or not hasattr(QtCore, 'PYQT_VERSION'):
        QtGui.QApplication.instance().exec_()

max_stars_repo_name: OREL-group/DevelopmentalBVs

max_forks_repo_name: OREL-group/DevelopmentalBVs


text:
import cv2                 # working with, mainly resizing, images
import numpy as np         # dealing with arrays
import os                  # dealing with directories
from random import shuffle # mixing up or currently ordered data that might lead our network astray in training.

path='data'

IMG_SIZE = 96

def create_train_data():
    training_data = []
    label = 0
    for (dirpath,dirnames,filenames) in os.walk(path):
        for dirname in dirnames:
            print(dirname)
            for(direcpath,direcnames,files) in os.walk(path+"/"+dirname):
                for file in files:
                        actual_path=path+"/"+dirname+"/"+file
                        print(files)
                        # label=label_img(dirname)
                        path1 =path+"/"+dirname+'/'+file
                        img=cv2.imread(path1,cv2.IMREAD_GRAYSCALE)#cv2.imread() method loads an image from the specified file.
                        img = cv2.resize(img, (IMG_SIZE,IMG_SIZE))# To resize an image, OpenCV provides cv2.resize() function.
                        training_data.append([np.array(img),label])
            label = label + 1
            print(label)
    shuffle(training_data)
    np.save('train_data.npy', training_data)
    print(training_data)
    return training_data

create_train_data()


text:
"""
(c) RIKEN 2016. All rights reserved.
Author: Keitaro Yamashita
This software is released under the new BSD License; see LICENSE.
"""
import re
import numpy
import sys

#float_or_1 = lambda x: 1 if x == "" else float(x)
float_or_1 = lambda x: (1,-1)[len(x)] if x == "" or x == "-" else float(x)

class Sensor:
    re_x = re.compile("([-0-9\.]*) *x")
    re_y = re.compile("([-+0-9\.]*) *y")

    def __init__(self, clen=None, res=None):
        for k in ("min_fs", "min_ss", "max_fs", "max_ss", "fs", "ss", "corner_x", "corner_y", "coffset"):
            setattr(self, k, None)

        self.clen = clen
        self.res = res
    # __init__()

    def set_info(self, key, valstr):
        if key == "min_fs":
            self.min_fs = int(float(valstr))
        elif key == "min_ss":
            self.min_ss = int(float(valstr))
        elif key == "max_fs":
            self.max_fs = int(float(valstr))
        elif key == "max_ss":
            self.max_ss = int(float(valstr))
        elif key == "fs":
            self.fs = [0., 0.]
            r_x = self.re_x.search(valstr)
            r_y = self.re_y.search(valstr)
            if r_x: self.fs[0] = float_or_1(r_x.group(1))
            if r_y: self.fs[1] = float_or_1(r_y.group(1))
            if not r_x and not r_y: print "Wrong parameter value for fs:", valstr
        elif key == "ss":
            self.ss = [0., 0.]
            r_x = self.re_x.search(valstr)
            r_y = self.re_y.search(valstr)
            if r_x: self.ss[0] = float_or_1(r_x.group(1))
            if r_y: self.ss[1] = float_or_1(r_y.group(1))
            if not r_x and not r_y: print >>sys.stderr, "Wrong parameter value for ss:", valstr
        elif key == "corner_x":
            self.corner_x = float(valstr)
        elif key == "corner_y":
            self.corner_y = float(valstr)
        elif key == "res":
            self.res = float(valstr)
        elif key == "clen":
            self.clen = float(valstr)
        elif key == "coffset":
            self.coffset = float(valstr)
        else:
            print >>sys.stderr, "Warning - Unknown paramter:", key
    # read_info()
# class Sensor

class Geomfile:
    def __init__(self, geom_in=None):
        self.clen = None
        self.res = None
        if geom_in is not None:
            self.read_file(geom_in)
    # __init__()

    def read_file(self, geom_in):
        self.sensors = {}

        for l in open(geom_in):
            if ";" in l: l = l[:l.find(";")]
            l = l.strip()

            if "=" in l:
                sp = map(lambda x:x.strip(), l.split("="))
                if len(sp) > 2:
                    print "Warning: more than one = in this line. Ignored:", l
                    continue

                lhs, rhs = sp
                if lhs == "clen":
                    self.clen = rhs #float
                elif lhs == "res":
                    self.res = float(rhs)
                elif lhs == "adu_per_eV":
                    self.adu_per_eV = rhs #float
                elif lhs == "max_adu":
                    self.max_adu = rhs # float
                elif "/" in lhs:
                    sp = lhs.split("/")
                    if len(sp) > 2:
                        print >>sys.stderr, "Warning: more than one / in left hand. Ignored:", l
                        continue

                    sid, lhs2 = sp
                    if sid not in self.sensors:
                        self.sensors[sid] = Sensor(clen=self.clen, res=self.res) # XXX res and clen must be defined first (before the sensor defs; if sensor defs don't have clen nor res)

                    self.sensors[sid].set_info(lhs2, rhs)
    # read_file()

    def get_extent(self):
        inf = float("inf")
        fmin, fmax, smin, smax = inf, -inf, inf, -inf
        for sid in self.sensors:
            s = self.sensors[sid]
            c = numpy.array((s.corner_x, s.corner_y))
            fs, ss = numpy.array(s.fs), numpy.array(s.ss)
            #for x in xrange(s.max_fs-s.min_fs+1):
            #    for y in xrange(s.max_ss-s.min_ss+1):
            for x in (0, s.max_fs-s.min_fs):
                for y in (0, s.max_ss-s.min_ss):
                    v = c + fs * x + ss * y
                    fmin = min(fmin, v[0])
                    fmax = max(fmax, v[0])
                    smin = min(smin, v[1])
                    smax = max(smax, v[1])

        return fmin, fmax, smin, smax
    # get_extent()

    def get_image_extent(self):
        max_ss, max_fs = 0, 0
        for sid in self.sensors:
            s = self.sensors[sid]
            max_fs = max(max_fs, s.max_fs)
            max_ss = max(max_ss, s.max_ss)
        return max_fs, max_ss
    # get_image_extent()

    def calc_xy_from_fs_ss(self, fs, ss, panel=None, shifts=None):
        if panel is None:
            for sid in self.sensors:
                s = self.sensors[sid]
                if s.min_fs <= fs <= s.max_fs and s.min_ss <= ss <= s.max_ss:
                    panel = sid
                    break

        if panel is None:
            return float("nan"), float("nan")

        s = self.sensors[panel]
        fs, ss = fs - s.min_fs, ss - s.min_ss

        x = s.corner_x + s.fs[0]*fs + s.ss[0]*ss
        y = s.corner_y + s.fs[1]*fs + s.ss[1]*ss

        if shifts is not None: # shifts in meter
            x += shifts[0]*s.res
            y += shifts[1]*s.res

        return x,y
    # calc_xy_from_fs_ss()
# class Geomfile


text:
"""Try Gammapy config with astropy.config."""
from astropy.config import ConfigNamespace, ConfigItem

class LoggingConfig(ConfigNamespace):
    level = ConfigItem()
    filename = ConfigItem()

class ObservationConfig(ConfigNamespace):
    c1 = ConfigItem(cfgtype="int")
    c2 = ConfigItem(cfgtype="float")
    c3 = ConfigItem(cfgtype="string")

    log = LoggingConfig()

if __name__ == '__main__':
    c = ObservationConfig()
    print(c)
    print(c.c1)
    print(c.log)
    print(c.log.level)
    print(c.__dict__)

max_stars_repo_name: cdeil/gammapy-config

max_issues_repo_name: cdeil/gammapy-config

max_forks_repo_name: cdeil/gammapy-config

reason: from astropy


text:
'''
#                           This is an 80 character line                       #
This file is intended to track the movement of the ballistic or active particle
over time.

We want to know:
    -if reorientation allows for enhanced mobility in the HCP phase
    -the critical activity for intrusion into HCP phase
    -the critical lattice spacing
    -relationship between mobility and activity
'''

import sys

pe = float(sys.argv[1])         # activity of active particle
lat = float(sys.argv[2])        # lattice spacing of HCP phase

gsd_path = '/nas/longleaf/home/kolbt/programs/gsd/build'
sys.path.append(gsd_path)       # ensure gsd is in your python path
gsd_path = '/Users/kolbt/Desktop/compiled/gsd/build'
sys.path.append(gsd_path)       # ensure gsd is in your python path

import gsd
from gsd import hoomd
from gsd import pygsd

import freud
from freud import parallel
from freud import box
from freud import density
from freud import cluster

import numpy as np
import math
from scipy import stats

#import matplotlib
#matplotlib.use('Agg')
import matplotlib.pyplot as plt

def computeA(diameter):
    """Computes area of circle"""
    radius = diameter / 2.0
    return np.pi * (radius**2)

def getDistance(point1, point2x, point2y):
    """Find the distance between two points"""
    distance = np.sqrt((point2x - point1[0])**2 + (point2y - point1[1])**2)
    return distance

def computeTauPerTstep(epsilon, mindt=0.000001):
    """Read in epsilon, output tauBrownian per timestep"""
    kBT = 1.0
    tstepPerTau = float(epsilon / (kBT * mindt))
    return 1. / tstepPerTau

# Base filename
inFile = "active_in_bulk_pe" + str(pe) + "_lattice" + str(lat) + ".gsd"
outFile = "active_in_bulk_pe" + str(pe) + "_lattice" + str(lat) + ".txt"
f = hoomd.open(name=inFile, mode='rb')

dump = int(f.__len__())         # get number of timesteps dumped
start = 0                       # gives first frame to read
end = dump                      # gives last frame to read

hcpPos = np.zeros((end), dtype=np.ndarray)
actPos = np.zeros((end), dtype=np.ndarray)          # array of active positions
types = np.zeros((end), dtype=np.ndarray)           # particle types
box_data = np.zeros((1), dtype=np.ndarray)          # box dimensions
timesteps = np.zeros((end), dtype=np.float64)       # timesteps

# Access and read .gsd data
with hoomd.open(name=inFile, mode='rb') as t:
    snap = t[0]
    box_data = snap.configuration.box
    for iii in range(start, end):
        snap = t[iii]                                   # snapshot of frame
        actPos[iii] = snap.particles.position[0]        # get active particle positions
        timesteps[iii] = snap.configuration.step        # get timestep

# Normalize timesteps
tauPerDT = computeTauPerTstep(epsilon=1.)
# Compute smallest unit to enforce larger timeaverages are multiples of it
dtHalfTauR = int(1. / (6. * tauPerDT))
dtInTauR = dtHalfTauR * 2
dtInTauB = dtHalfTauR * 6
timesteps -= timesteps[0]
dt = timesteps[-1] - timesteps[-2]
# Brownian time
timesteps *= tauPerDT
# Rotational time units
#timesteps *= (3. * tauPerDT)
dtau = timesteps[-1] - timesteps[-2]

# Box dimensions
x_box = box_data[0]
y_box = box_data[1]
hx_box = x_box / 2.0
hy_box = y_box / 2.0
a_box = x_box * y_box

# Extent of HCP phase
#instV = np.zeros((end), dtype=np.float64)
instV = []
halfV = []
taurV = []
taubV = []
# Compute the MSD in the HCP phase
for i in range(start + 1, end):
    # Compute the distance traveled between timesteps (include periodicity)
    dx = abs(actPos[i][0] - actPos[i-1][0])
    dy = abs(actPos[i][1] - actPos[i-1][1])
    if dx > hx_box:
        dx = x_box - dx
    if dy > hy_box:
        dy = y_box - dy
    instV.append( (np.sqrt((dx**2)+(dy**2))) / dtau )

    # Is this a multiple of a larger timescale (0.5 * tauR)?
    if i % dtHalfTauR == 0:
        print("At half tauR")
        ind = i - dtHalfTauR
        dx = abs(actPos[i][0] - actPos[ind][0])
        dy = abs(actPos[i][1] - actPos[ind][1])
        if dx > hx_box:
            dx = x_box - dx
        if dy > hy_box:
            dy = y_box - dy
        halfV.append( (np.sqrt((dx**2)+(dy**2))) / (dtau * dtHalfTauR) )

        # Is this a multiple of tauR
        if i % dtInTauR == 0:
            print("At tauR")
            ind = i - dtInTauR
            dx = abs(actPos[i][0] - actPos[ind][0])
            dy = abs(actPos[i][1] - actPos[ind][1])
            if dx > hx_box:
                dx = x_box - dx
            if dy > hy_box:
                dy = y_box - dy
            taurV.append( (np.sqrt((dx**2)+(dy**2))) / (dtau * dtInTauR) )

            # Is this a multiple of tauB
            if i % dtInTauB == 0:
                print("At tauB")
                ind = i - dtInTauB
                dx = abs(actPos[i][0] - actPos[ind][0])
                dy = abs(actPos[i][1] - actPos[ind][1])
                if dx > hx_box:
                    dx = x_box - dx
                if dy > hy_box:
                    dy = y_box - dy
                taubV.append( (np.sqrt((dx**2)+(dy**2))) / (dtau * dtInTauB) )

## We can plot the free and dense phase active particle velocity
#plt.plot(timesteps[start+1:end], instV, c='b', ls='--', label=r'$\nu_{inst.}$')
##plt.plot(timesteps[start+1:end], halfV, c='r', ls='--', label=r'$\nu_{\tau_{r}/2}$')
##plt.plot(timesteps[start+1:end], taurV, c='g', ls='--', label=r'$\nu_{\tau_{r}}$')
##plt.plot(timesteps[start+1:end], taubV, c='k', ls='--', label=r'$\nu_{\tau_{B}}$')
#plt.xlabel(r'Time $(\tau_{B})$')
#plt.ylabel(r'Velocity $\left(\frac{d\sigma}{d\tau_{B}}\right)$')
#plt.legend(loc=1)
#plt.xlim(0, max(timesteps))
#plt.ylim(0, pe)
#plt.show()

# Compute and overlay the average
avgInstV = float(sum(instV)) / float(len(instV))
avgHalfV = float(sum(halfV)) / float(len(halfV))
avgTaurV = float(sum(taurV)) / float(len(taurV))
avgTaubV = float(sum(taubV)) / float(len(taubV))

## Plot the data
#mysz = 15.
#x, y, z = zip(*actPos[start:end])
## Plot the time resolved active particle
#plt.scatter(x, y, c=(timesteps[start:end]/max(timesteps)), s=mysz, cmap='jet')
#plt.xlim(-hx_box, hx_box)
#plt.ylim(-hy_box, hy_box)
#ax = plt.gca()
#ax.set_aspect('equal')
#cbar = plt.colorbar(ticks=[], fraction=0.02675, pad=0.05, label='Time')
#plt.text(1.085, 0.015, r'$\tau_{initial}$', transform=ax.transAxes, fontsize=14)
#plt.text(1.085, 0.95, r'$\tau_{final}$', transform=ax.transAxes, fontsize=14)
#plt.show()

# Make text file for relevant quantities
f = open(outFile, 'w')
f.write('Activity'.center(15) + ' ' +\
        'Lattice'.center(15) + ' ' +\
        'Instant_v'.center(15) + ' ' +\
        'half_taur_v'.center(15) + ' ' +\
        'taur_v'.center(15) + ' ' +\
        'taub_v'.center(15) + '\n')
f.close()

# Populate text file
f = open(outFile, 'a')
f.write('{0:.1f}'.format(pe).center(15) + ' ' +\
        '{0:.2f}'.format(lat).center(15) + ' ' + \
        '{0:.4f}'.format(avgInstV).center(15) + ' ' + \
        '{0:.4f}'.format(avgHalfV).center(15) + ' ' + \
        '{0:.4f}'.format(avgTaurV).center(15) + ' ' + \
        '{0:.4f}'.format(avgTaubV).center(15) + '\n')
f.close()

max_stars_repo_name: kolbt/whingdingdilly

max_issues_repo_name: kolbt/whingdingdilly

max_forks_repo_name: kolbt/whingdingdilly

reason: import numpy,from scipy