text:
import numpy as np
import os
import argparse
import glob
from joblib import dump, load

from PIL import Image
from lapjv import lapjv
from sklearn.manifold import TSNE
from scipy.spatial import distance_matrix
# from scipy.misc import imsave
from sklearn.decomposition import PCA
from scipy.spatial.distance import cdist
from tensorflow.python.keras.applications.vgg16 import VGG16
from keras.applications.vgg16 import preprocess_input
from tensorflow.python.keras.preprocessing import image
from tensorflow.python.keras.models import Model, Sequential
from tensorflow.python.keras.layers import GlobalAveragePooling2D

GENERATE_BIG_PICTURE = True
TRANSFORM_FROM_CNN_DIM_TO_SOUND_DIM = True
# True: transform form 512 to 5 using PCA. Otherwise, using random matrix
TRANSFORM_USING_PCA = True
# True: compute the 512d vector from de original image. Otherwise, from
# Black/White
WORK_WITH_ORIGINAL_IMAGES = True
CNN_DIM = 512
SOUND_DIM = 5

OUT_DIM = 6  # number of images in a row/column in output image
IN_DIR = './data/camera_view/video_frames/'  # source directory for images
OUT_DIR = './data/'  # destination directory for output image

PCA_PATH = './data/pca.joblib'

to_plot = np.square(OUT_DIM)
out_res = 224  # width/height of images in output square image
perplexity = 10  # TSNE perplexity
tsne_iter = 5000000  # number of iterations in TSNE algorithm

if OUT_DIM == 1:
    raise ValueError("Output grid dimension 1x1 not supported.")

if __name__ == '__main__':
    if not os.path.exists(OUT_DIR):
        raise argparse.ArgumentTypeError(
            "'{}' not a valid directory.".format(OUT_DIR))

    if not os.path.exists(IN_DIR):
        raise argparse.ArgumentTypeError(
            "'{}' not a valid directory.".format(IN_DIR))

def build_model():
    """
    This function load a trained VGG16 CNN excluding the final Dense layer. Then it adds one more layer
    (GlobalAveragePooling2D) to transform the last volume (?,?,512) to a 512 dimensional vector
    :return: the model
    """
    base_model = VGG16(
        weights='imagenet',
        input_shape=(
            224,
            224,
            3),
        include_top=False)  # do not load final Dense Layers
    top_model = Sequential()
    # Transform Conv+Pooling in a vector (512 dimensions)
    top_model.add(GlobalAveragePooling2D())
    return Model(inputs=base_model.input, outputs=top_model(base_model.output))

def load_img(input_dir):
    """
    This function load all PNG images stored in 'input_dir'
    :param input_dir: the directory where the images are stored
    :return: a list with the images and a list with the names of the files
    """
    pred_img = sorted(glob.glob(input_dir + "/*.png"))
    img_collection = []
    file_names = []
    for idx, img in enumerate(pred_img):
        print("Loading image", idx + 1, ":", img)
        file_names.append(img)
        img_collection.append(
            image.load_img(
                img, target_size=(
                    out_res, out_res)))
    if np.square(OUT_DIM) > len(img_collection):
        raise ValueError(
            "Cannot fit {} images in {}x{} grid".format(
                len(img_collection), OUT_DIM, OUT_DIM))
    return img_collection, file_names

def binarize_array(numpy_array, threshold=200):
    """Binarize an image in a numpy array."""
    for i in range(len(numpy_array)):
        for j in range(len(numpy_array[0])):
            if numpy_array[i][j] > threshold:
                numpy_array[i][j] = 255
            else:
                numpy_array[i][j] = 0
    return numpy_array

def get_activations(model, img_collection, file_names, save_files=False):
    """
    This function computes one vector for each image using the CNN model and store them in 'activations'.
    Each column of 'activations' corresponds with the vector of one image.
    Those vectors are stored in 'out_vectors.csv'.
    This function also produces the file 'out_distances.csv' where the pairwise distances between those vectos are
    write down.
    :param model: the trained CNN model
    :param img_collection: the list with the loaded images
    :param file_names: the list with the corresponding file names of the images
    :return: a list with the vectors, the files described above and the PIL images in B/W
    """
    activations = []
    header = ''
    img_collection_bn = []
    for idx, img in enumerate(img_collection):
        # if idx == to_plot:
        #     break
        print("Processing image", idx + 1, ":", img)
        img = img.resize((out_res, out_res), Image.ANTIALIAS)
        img_bw = img.convert('L')  # convert image to monochrome
        xbw = image.img_to_array(img_bw)
        xbw = np.reshape(xbw, (224, 224))
        xbw = binarize_array(xbw)  # filtering
        img_bw = Image.fromarray(xbw)
        img_bw = img_bw.convert('RGB')  # convert image to RGB
        if save_files:
            img_bw.save(file_names[idx] + '.BW.jpg')
        img_collection_bn.append(img_bw)
        if WORK_WITH_ORIGINAL_IMAGES:
            x = image.img_to_array(img)
        else:
            x = image.img_to_array(img_bw)
        x = preprocess_input(x)
        x = np.expand_dims(x, axis=0)
        activations.append(np.squeeze(model.predict(x)))
        header = header + os.path.basename(file_names[idx]).split('.')[0] + ';'
    distances = distance_matrix(activations, activations)
    if save_files:
        np.savetxt("out_vectors.csv", list(
            map(list, zip(*activations))), delimiter=";", header=header)
        np.savetxt(
            "out_distances.csv",
            distances,
            delimiter=";",
            header=header)
    return activations, header, img_collection_bn

def generate_tsne(activations):
    """
    This function maps the vectors in 'activations' in a 2 dimensional space where those vectors that similar must
    appear close. It uses the algorithm TSNE.
    :param activations: a list with vectors
    :return: the representation of those vector in that 2 dimensional space
    """
    tsne = TSNE(
        perplexity=perplexity,
        n_components=2,
        init='random',
        n_iter=tsne_iter)
    x_2d = tsne.fit_transform(np.array(activations)[0:to_plot, :])
    x_2d -= x_2d.min(axis=0)
    x_2d /= x_2d.max(axis=0)
    return x_2d

def save_tsne_grid(
        img_collection,
        x_2d,
        output_res,
        output_dim,
        output_dir,
        out_name):
    """
    This function creates an image made of the original images placed relative to what is indicated in x_2d
    :param img_collection: the list with the loaded images
    :param x_2d: the 2d coordinates obtained for the images using TSNE
    :param output_res: resolution of the images
    :param output_dim: the composed image will be created using (output_dim x output_dim) images
    :param output_dir: destination directory for output image
    :param out_name: name of the output image
    :return: This function produces a JPG image in the 'output_dir' directory
    """
    grid = np.dstack(np.meshgrid(np.linspace(0, 1, output_dim),
                                 np.linspace(0, 1, output_dim))).reshape(-1, 2)
    cost_matrix = cdist(grid, x_2d, "sqeuclidean").astype(np.float32)
    cost_matrix = cost_matrix * (100000 / cost_matrix.max())
    row_asses, col_asses, _ = lapjv(cost_matrix)
    grid_jv = grid[col_asses]
    out = np.ones((output_dim * output_res, output_dim * output_res, 3))

    for pos, img in zip(grid_jv, img_collection[0:to_plot]):
        h_range = int(np.floor(pos[0] * (output_dim - 1) * output_res))
        w_range = int(np.floor(pos[1] * (output_dim - 1) * output_res))
        out[h_range:h_range +
            output_res, w_range:w_range +
            output_res] = image.img_to_array(img)

    im = image.array_to_img(out)
    im.save(output_dir + out_name, quality=100)

def generate_M_and_MG(seed, deviation):
    """
    This function generates random matrices M and MG
    :param seed: seed for the random number generator
    :param deviation: for generate following a normal distribution. Small deviation in order to generate numbers
    close to cero
    :return: matrices M and MG
    """
    np.random.seed(seed=seed)
    M = np.random.uniform(low=-1, high=1, size=(CNN_DIM, SOUND_DIM))
    MG = np.random.normal(0, scale=deviation, size=(CNN_DIM, SOUND_DIM))
    return M, MG

def get_M_and_MG_from_file():
    """
    This function loads both matrices from files
    :return: matrices M and MG
    """
    M = np.load('matrixUniform.npy')
    MG = np.load('matrixGaussian.npy')
    return M, MG

def get_M_from_file_and_generate_MG(seed, deviation):
    """
    This function loads M from file and generates MG
    :param seed: seed for the random number generator
    :param deviation: for generate following a normal distribution. Small deviation in order to generate numbers
    close to cero
    :return: matrices M and MG
    """
    np.random.seed(seed=seed)
    M = np.load('matrixUniform.npy')
    MG = np.random.normal(0, scale=deviation, size=(CNN_DIM, SOUND_DIM))
    return M, MG

def sigmoid(mat, coef):
    """
    This function uses sigmoid function to map all values in 'mat' in the range [0,1]
    :param mat: a list of vectors. In this case each vector represents one image
    :param coef: modify the slope of the sigmoid: big values of coef make the function shrink
    :return: the matrix with all the values in [0,1]
    """
    return 1 / (1 + np.exp(-mat * coef))  # Sigmoid function

def main():
    model = build_model()
    model.summary()

    img_collection, names_of_file = load_img(IN_DIR)

    activations, header, img_coll_bn = get_activations(
        model, img_collection, names_of_file, True)

    if TRANSFORM_FROM_CNN_DIM_TO_SOUND_DIM:
        if TRANSFORM_USING_PCA:
            # you can comment the following 3 lines when you have trained the
            # PCA previously
            pca = PCA(n_components=SOUND_DIM)
            act_5dim = pca.fit_transform(activations)
            dump(pca, PCA_PATH)  # store the trained pca

            # uncomment the following 2 lines if you have trained previously de PCA
            # pca = load(PCA_PATH)
            # act_5dim = pca.transform(activations)

            # if you want to add some gaussian noise to the 5dim vectors you can uncomment the following 2 lines
            # MG = np.random.normal(0, scale=0.1, size=act_5dim.shape)
            # act_5dim = act_5dim + MG
        else:
            # M: is a random (uniform dist.) matrix that projects CNN_DIM dimensional vectors in SOUND_DIM dim
            # MG: is a random (normal dist.) matrix if you want to slightly modify M
            # comment/uncomment next lines to obtain what you want
            # change the seed to obtain different matrices
            M, MG = generate_M_and_MG(seed=2022, deviation=0.01)
            # M, MG = get_M_and_MG_from_file()
            # M, MG = get_M_from_file_and_generate_MG(seed=2019, deviation=0.01)

            # you can modify M a little bit uncommenting next line
            # M = M + MG

            # if you want to store the matrices in files you need to uncomment next lines
            # Be careful!!! if the matrices already exist then these
            # instructions will override them
            np.save('matrixUniform', M)
            np.save('matrixGaussian', MG)

            act_5dim = activations @ M  # matrix multiplication

        print(act_5dim)
        # activations = sigmoid(act_5dim, coef=0.05)  # Sigmoid function
        # activations = act_5dim
        mins = np.min(act_5dim, 0)
        maxs = np.max(act_5dim, 0)
        print("maxs-mins:")
        print((maxs - mins))
        activations = (act_5dim - mins) / (maxs - mins)
        print(activations)

        # store in files the new 5dim vectors and the pairwise distances
        np.savetxt("out_vectors_5d.csv", list(
            map(list, zip(*activations))), delimiter=";", header=header)
        distances = distance_matrix(activations, activations)
        np.savetxt(
            "out_distances_5d.csv",
            distances,
            delimiter=";",
            header=header)

    if GENERATE_BIG_PICTURE:
        # Birk, from here it is only for generating the image. Actually, you do not need it but it is very cool to see
        # it and it is useful in order to understand the nature of the vectors
        # that the CNN produce
        print("Generating 2D representation.")
        x_2dim = generate_tsne(activations)
        print("Generating image grid.")
        save_tsne_grid(img_collection, x_2dim, out_res, OUT_DIM,
                       OUT_DIR, out_name='out_image_original.jpg')
        save_tsne_grid(
            img_coll_bn,
            x_2dim,
            out_res,
            OUT_DIM,
            OUT_DIR,
            out_name='out_image_BW.jpg')

if __name__ == '__main__':
    main()

reason: import numpy,from scipy


text:
import numpy as np
from scipy.special import gammainc
from skimage.transform import resize

# Uniform sampling in a hyperspere
# Based on Matlab implementation by Roger Stafford
# Can be optimized for Bridson algorithm by excluding all points within the r/2 sphere
def hypersphere_volume_sample(center, radius, k=1, rng_new=np.random.default_rng):
    ndim = center.size
    x = rng_new.normal(size=(k, ndim))
    ssq = np.sum(x ** 2, axis=1)
    fr = radius * gammainc(ndim / 2, ssq / 2) ** (1 / ndim) / np.sqrt(ssq)
    frtiled = np.tile(fr.reshape(k, 1), (1, ndim))
    p = center + np.multiply(x, frtiled)
    return p

# Uniform sampling on the sphere's surface
def hypersphere_surface_sample(center, radius, k=1):
    ndim = center.size
    vec = np.random.standard_normal(size=(k, ndim))
    vec /= np.linalg.norm(vec, axis=1)[:, None]
    p = center + np.multiply(vec, radius)
    return p

def squared_distance(p0, p1):
    return np.sum(np.square(p0 - p1))

def Bridson_sampling(dims=np.array([1.0, 1.0]), radius=0.05, k=30, hypersphere_sample=hypersphere_volume_sample, rng_new=None):
    # References: Fast Poisson Disk Sampling in Arbitrary Dimensions
    #             Robert Bridson, SIGGRAPH, 2007

    ndim = dims.size

    # size of the sphere from which the samples are drawn relative to the size of a disc (radius)
    sample_factor = 2
    if hypersphere_sample == hypersphere_volume_sample:
        sample_factor = 2

    # for the surface sampler, all new points are almost exactly 1 radius away from at least one existing sample
    # eps to avoid rejection
    if hypersphere_sample == hypersphere_surface_sample:
        eps = 0.001
        sample_factor = 1 + eps

    def in_limits(p):
        return np.all(np.zeros(ndim) <= p) and np.all(p < dims)

    # Check if there are samples closer than "squared_radius" to the candidate "p"
    def in_neighborhood(p, n=2):
        indices = (p / cellsize).astype(int)
        indmin = np.maximum(indices - n, np.zeros(ndim, dtype=int))
        indmax = np.minimum(indices + n + 1, gridsize)

        # Check if the center cell is empty
        if not np.isnan(P[tuple(indices)][0]):
            return True
        a = []
        for i in range(ndim):
            a.append(slice(indmin[i], indmax[i]))
        with np.errstate(invalid='ignore'):
            if np.any(np.sum(np.square(p - P[tuple(a)]), axis=ndim) < squared_radius):
                return True

    def add_point(p):
        points.append(p)
        indices = (p / cellsize).astype(int)
        P[tuple(indices)] = p

    cellsize = radius / np.sqrt(ndim)
    gridsize = (np.ceil(dims / cellsize)).astype(int)

    # Squared radius because we'll compare squared distance
    squared_radius = radius * radius

    # Positions of cells
    P = np.empty(np.append(gridsize, ndim), dtype=np.float32)  # n-dim value for each grid cell
    # Initialise empty cells with NaNs
    P.fill(np.nan)

    points = []
    add_point(rng_new.uniform(np.zeros(ndim), dims))
    while len(points):
        i = rng_new.integers(len(points))
        p = points[i]
        del points[i]
        Q = hypersphere_sample(np.array(p), radius * sample_factor, k, rng_new)
        for q in Q:
            if in_limits(q) and not in_neighborhood(q):
                add_point(q)
    return P[~np.isnan(P).any(axis=ndim)]

def create_circular_mask(h, w, center=None, radius=None):
    if center is None:  # use the middle of the image
        center = (int(w / 2), int(h / 2))
    if radius is None:  # use the smallest distance between the center and image walls
        radius = min(center[0], center[1], w - center[0], h - center[1])

    Y, X = np.ogrid[:h, :w]
    dist_from_center = np.sqrt((X - center[0]) ** 2 + (Y - center[1]) ** 2)

    mask = (dist_from_center <= radius).astype(float)
    return mask

class CircularPoissonMaskGenerator:
    def __init__(self, shape: np.array, central_sampling: bool, accel: float, radius: float = 0.0125, rng_new=None):
        if rng_new is None:
            rng_new = np.random.default_rng(42)
        self.shape = shape
        self.central_sampling = central_sampling
        self.accel = accel
        self.radius = radius
        self.mask = np.zeros(tuple(self.shape))
        self.rng_new = rng_new

    def __call__(self):
        while True:
            shape_bs = self.shape / max(self.shape)
            bs = Bridson_sampling(shape_bs, radius=self.radius, rng_new=self.rng_new)
            bs = (bs * max(self.shape)).astype(int)
            self.mask *= 0
            if self.central_sampling:
                cent_x = round(self.shape[0] // 2)
                cent_y = round(self.shape[1] // 2)
                perc_x = round(self.shape[0] * 0.15)
                perc_y = round(self.shape[1] * 0.05)
                self.mask[cent_x - perc_x:cent_x + perc_x, cent_y - perc_y:cent_y + perc_y] = 1
            self.mask[bs[:, 0], bs[:, 1]] = 1
            cm = create_circular_mask(100, 100)
            cm = resize(cm, tuple(self.shape))
            self.mask *= cm
            R = 1 / (np.sum(self.mask) / np.prod(self.mask.shape) + 1e-8)
            if abs(R - self.accel) < 0.1:
                return self.mask
            elif R > self.accel:
                self.radius -= 0.0001
            elif R < self.accel:
                self.radius += 0.0001

if __name__ == "__main__":
    import matplotlib.pyplot as plt

    gen = CircularPoissonMaskGenerator(np.array([32, 148]), True, 6, rng_new=np.random.default_rng())
    mask1 = gen()
    mask2 = gen()
    plt.subplot(1, 3, 1)
    plt.imshow(mask1)
    plt.subplot(1, 3, 2)
    plt.imshow(mask2)
    plt.subplot(1, 3, 3)
    plt.imshow(abs(mask1 - mask2))
    plt.savefig("test.png")

ext: py

max_stars_repo_name: lukasfolle/fastMRI

max_issues_repo_name: lukasfolle/fastMRI

max_forks_repo_name: lukasfolle/fastMRI


text:
from __future__ import absolute_import, print_function, division
import h5py
import logging
import numpy as np
import os.path
import pandas as pd
import re
import yaml

from superman.dataset import (
    NumericMetadata, PrimaryKeyMetadata, CompositionMetadata, DateMetadata)

from . import web_datasets
from .web_datasets import WebLIBSDataset, WebVectorDataset, WebTrajDataset

def load_datasets(config_fh, custom_loaders, public_only=False, user_added=False):
  config = yaml.safe_load(config_fh)

  for kind, entries in config.items():
    if entries:
      for name, info in entries.items():
        # skip this entry if it shouldn't be included
        is_public = info.get('public', True)
        if public_only and not is_public:
          continue

        if 'files' in info:
          files = info['files']
        else:
          files = [info['file']]

        if 'loader' in info:
          # look up the loader function from the module namespace
          loader_fn = getattr(custom_loaders, info['loader'])
        else:
          # construct a loader from the meta_mapping and the default template
          meta_mapping = [(k, getattr(web_datasets, cls), mname)
                          for k, cls, mname in info.get('metadata', [])]
          if info.get('vector', False):
            loader_fn = _generic_vector_loader(meta_mapping)
          else:
            loader_fn = _generic_traj_loader(meta_mapping)

        if kind == 'LIBS':
          ds = WebLIBSDataset(name, loader_fn, *files)
        elif info.get('vector', False):
          ds = WebVectorDataset(name, kind, loader_fn, *files)
        else:
          ds = WebTrajDataset(name, kind, loader_fn, *files)

        if 'description' in info:
          ds.description = info['description']
        if 'urls' in info:
          ds.urls = info['urls']
        ds.is_public = is_public
        ds.user_added = user_added

def try_load(filepath, data_name):
  logging.info('Loading %s data...' % data_name)
  if not os.path.exists(filepath):
    logging.warning('Data file for %s not found.' % data_name)
    return None

  load_fn = (np.load if filepath.endswith('.npz') else
             lambda f: h5py.File(f, mode='r'))
  try:
    return load_fn(filepath)
  except IOError as e:
    logging.warning('Failed to load %s data!' % data_name)
    logging.warning(str(e))
    return None

def _generic_traj_loader(meta_mapping):
  """Creates a loader function for a standard HDF5 file representing a
  trajectory dataset. The HDF5 structure is expected to be:
   - /meta/pkey : an array of keys, used to address individual spectra
   - /spectra/[pkey] : a (n,2) trajectory spectrum
   - /meta/foobar : (optional) metadata, specified by the meta_mapping
  """
  def _load(ds, filepath):
    data = try_load(filepath, str(ds))
    if data is None:
      return False
    meta = data['/meta']
    kwargs = {}
    for key, cls, display_name in meta_mapping:
      if key not in meta:
        continue
      m = meta[key]
      if cls is DateMetadata:
        m = pd.to_datetime(np.array(m))
      elif cls is PrimaryKeyMetadata:
        assert key == 'pkey'
        continue
      safe_key = re.sub(r'[^a-z0-9_-]', '', key, flags=re.I)
      kwargs[safe_key] = cls(m, display_name=display_name)
    ds.set_data(meta['pkey'], data['/spectra'], **kwargs)
    return True
  return _load

def _generic_vector_loader(meta_mapping):
  """Creates a loader function for a standard HDF5 file representing a
  vector dataset. The HDF5 structure is expected to be:
   - /meta/waves : length-d array of wavelengths
   - /spectra : (n,d) array of spectra
   - /meta/foobar : (optional) metadata, specified by the meta_mapping
   - /composition/[name] : (optional) composition metadata
  """
  def _load(ds, filepath):
    data = try_load(filepath, str(ds))
    if data is None:
      return False
    meta = data['/meta']
    kwargs = {}
    for key, cls, display_name in meta_mapping:
      if key not in meta:
        continue
      if cls is PrimaryKeyMetadata:
        kwargs['pkey'] = cls(meta[key])
      elif cls is DateMetadata:
        kwargs[key] = cls(pd.to_datetime(np.array(meta[key])),
                          display_name=display_name)
      else:
        kwargs[key] = cls(meta[key], display_name=display_name)
    if '/composition' in data:
      comp_meta = {name: NumericMetadata(arr, display_name=name) for name, arr
                   in data['/composition'].items()}
      kwargs['Composition'] = CompositionMetadata(comp_meta)
    ds.set_data(meta['waves'], data['/spectra'], **kwargs)
    return True
  return _load

reason: import numpy


text:
# --------------
# Instructions

# Different functions that you would require to define for this project has been mentioned in the code block. All the parameters and the task, a function would do, have been mentioned there.

#Importing the modules
import pandas as pd
import numpy as np
from scipy.stats import mode

# Load the weather_2012 data csv file and store it in weather variable. The path of the dataset has been stored in the variable path for you.

weather = pd.read_csv(path)

print(weather.head(5))

# Check the categorical and numerical variables. You can check it by calling categorical and numerical functions.

weather.select_dtypes("object").columns

print(weather.dtypes)

print(weather.select_dtypes("object"))

# for numerical variables

print(weather.select_dtypes("float"))

print(weather.select_dtypes("int"))

weather["Wind Spd (km/h)"].unique()

# Check the distribution of a specific value like the number of times the weather was exactly Cloudy in the given column. Feel free to check on other values. You can check it by calling the function clear with respective parameters.

weather["Weather"].value_counts()['Clear']

weather["Weather"].value_counts()['Cloudy']

# Find the number of times when the wind speed was exactly 4 km/h

weather["Wind Spd (km/h)"].value_counts()[4]

# By using the index of the value or name of the value you can check the number of counts. Now suppose you want to check some instances based on a specific condition like when the wind speed was above 35 and visibility was 25. You can directly check it by calling the function instances_based_condition with respective parameters and store the resulting dataframe in wind_speed_35_vis_25.

wind_speed_35_vis_25 = weather[(weather['Wind Spd (km/h)'] > 35) & (weather['Visibility (km)']== 25)]

print(wind_speed_35_vis_25)

# You have temperature data and want to calculate the mean temperature recorded by month. You can generate a pivot table that contains the aggregated values(like mean, max, min, sum, len) recorded by month.

#You can call the function agg_values_ina_month with respective parameters.

weather['Date/Time'] = pd.to_datetime(weather['Date/Time'])

mean_dew_temperature = weather.pivot_table(values='Dew Point Temp (C)', index=weather['Date/Time'].dt.month, aggfunc=np.mean)

print(mean_dew_temperature)

# To groupby based on a column like you want to groupby on Weather column and then aggregate the mean values of each column for different types of weather using mean.
# You can call the function group_values and store the resulting dataframe in mean_weather. Feel free to try on different aggregated functions like max, min, sum, len

mean_weather = weather.groupby("Weather").agg(np.mean)

print(mean_weather)

max_stars_repo_name: Niteshnupur/analyzing-weather-dataset

max_issues_repo_name: Niteshnupur/analyzing-weather-dataset

max_forks_repo_name: Niteshnupur/analyzing-weather-dataset


text:
import pandas as pd
from scipy.stats import chi2
import plotly
import plotly.graph_objs as go

def kruskal_wallis_test(df, column1_name, column2_name, alpha):

	k = df[column2_name].nunique()

	N = df[column2_name].count()

	df['rank'] = df[column1_name].rank(ascending=True)
	table1 = df.groupby(column2_name).sum()
	table2 = df.groupby(column2_name).count()

	sub_component = 0
	for i in range(0, k):
		sub_component = (table1.iloc[i, 1] ** 2 / table2.iloc[i, 1]) + sub_component

	test_statistic = ((12 / (N * (N + 1))) * sub_component) - 3 * (N + 1)

	degrees_of_freedom = k - 1
	a = alpha / 100
	chi_critical_value = round(chi2.isf(q=a, df=degrees_of_freedom), 2)
	p_value = chi2.sf(test_statistic, degrees_of_freedom)

	print('\n Rejection Criteria: Reject null hypothesis at', alpha, '% level of significance '
			'if Test Statistic is greater than or equal ',chi_critical_value, '.')

	table = {'Variable': column1_name + ' grouped by ' + column2_name, 'Test Statistic': round(test_statistic,4),
			   'Critical Value': chi_critical_value,'P value': p_value}

	trace = go.Table(
		header=dict(values=list(table.keys()),
					fill=dict(color='#C2D4FF'),
					align=['left'] * 5),
		cells=dict(values=list(table.values()),
				   fill=dict(color='#F5F8FF'),
				   align=['left'] * 5))

	data = [trace]

	plotly.offline.plot({'data': data}, filename='Table.html')

"""Applying Kruskal Wallis Test for Tensile Strength data set"""

data = pd.read_csv('Tensile Strength.csv')
kruskal_wallis_test(data, column1_name='Tensile Strength', column2_name='Mixing Technique', alpha=5)

max_stars_repo_path: Kruskal Wallis Test.py

max_stars_repo_name: Indkul/Non-parametric-test

max_issues_repo_path: Kruskal Wallis Test.py

max_issues_repo_name: Indkul/Non-parametric-test

max_forks_repo_path: Kruskal Wallis Test.py

max_forks_repo_name: Indkul/Non-parametric-test

reason: from scipy


text:
# ======================================================================
#     Created by Josh Aberdeen, Cameron Gordon, Patrick O'Callaghan 11/2021
# ======================================================================

import numpy as np

# ======================================================================
## Verbosity of print output
verbose = True
economic_verbose = True
# ======================================================================
# number of goods of the model
n_agt = 2  ## for Cai-Judd this is number of regions
# Number of paths
No_samples = 10 * n_agt
# ======================================================================
## Control of paths
# To start from scratch, set numstart = 0.
# Otherwise set numstart equal to previous numits. (Equivalently,
# set numstart equal to the last path.)
numstart = 0
# how many i_pths
fthrits = 4
numits = numstart + fthrits
# ======================================================================
# Number of test points to compute the error in postprocessing
# No_samples_postprocess = 20 ##not needed as we use all paths generated now

# ======================================================================
# length_scale_bounds=(10e-9,10e10) ##not needed: for GP

ipopt_tol = 1e-1 ##not needed: for GP
# n_restarts_optimizer=10 ##not needed: for GP
filename = "paths/path_number_"
# folder with the restart/result files

# ======================================================================
# Model Parameters

p_01 = 0.01 # transition probability from state 0 to 1
Delta = 30
Tstar = 20
beta = 0.99
# rho = 0.95
zeta1 = 1
zeta2 = 0.95
phi = 0.5 # adjustment cost multiplier
phik = 0.5 # weight of capital in production # alpha in CJ
phim = 0.5 # weight of intermediate inputs in production

gamma = 0.5 # power utility exponent
gammahat = 1 - 1/gamma
tau = np.ones(n_agt) # regional weights ### change to rho
delta = 0.025 # discount factor
eta = 0.5 #Frisch elasticity of labour supply
etahat = 1+1/eta
A = (1- (1-delta)*beta) / (phik * beta)
B = (1-phik)*A*(A-delta)**(-1/gamma)
xi = np.ones(n_agt) * 1 / n_agt
mu = np.ones(n_agt) * 1 / n_agt

# Ranges For States
kap_L = 0.1
kap_U = 10

#k0(j) = exp(log(kmin) + (log(kmax)-log(kmin))*(ord(j)-1)/(card(j)-1));
k_init = np.ones(n_agt)
for j in range(n_agt):
    k_init[j] = np.exp(np.log(kap_L) + (np.log(kap_U)-np.log(kap_L)) * j / (n_agt-1))
# ======================================================================

max_stars_repo_name: uq-aibe/spir-oz

max_issues_repo_name: uq-aibe/spir-oz

max_forks_repo_name: uq-aibe/spir-oz


text:
import pytest
import types
import numpy as np
from hcrystalball.model_selection import FinerTimeSplit

@pytest.mark.parametrize(
    "ts_data, expected_error",
    [
        ("series", None),
        ("series_with_NaN", None),
        ("series_with_Inf", None),
        ("series_with_name", None),
        ("series_with_index_name", None),
        ("dataframe", None),
        ("dataframe_with_name", None),
        ("dataframe_with_index_name", None),
        ("dataframe_multicolumn", None),
        ("dataframe_integer_index", None),
        ("empty_dataframe", ValueError),
        ("empty_series", ValueError),
    ],
    indirect=["ts_data"],
)
def test_cv_finertimesplit_split_pandas_container_data(ts_data, expected_error):

    n_splits = 2
    horizon = 3
    fts = FinerTimeSplit(n_splits=n_splits, horizon=horizon)
    if expected_error is None:
        result = fts.split(ts_data)
        assert isinstance(result, types.GeneratorType)
        result = list(result)
        assert len(result) == n_splits
        for i, isplit in enumerate(result):
            assert len(isplit) == 2
            assert len(isplit[0]) == len(ts_data) - (n_splits - i) * horizon
            assert len(isplit[1]) == horizon
            assert np.array_equal(isplit[0], np.arange(len(ts_data) - (n_splits - i) * horizon))
            assert np.array_equal(isplit[1], np.arange(horizon) + len(ts_data) - (n_splits - i) * horizon)
    else:
        with pytest.raises(expected_error):
            _ = list(fts.split(ts_data))

@pytest.mark.parametrize(
    "test_data, expected_error",
    [(np.arange(6), None), ([0, 1, 2, 3, 4, 5], None), ((0, 1, 2, 3, 4, 5), None), (13, TypeError)],
)
def test_cv_finertimesplit_split_input_data_types(test_data, expected_error):

    n_splits = 2
    horizon = 3
    fts = FinerTimeSplit(n_splits=n_splits, horizon=horizon)
    if expected_error is None:
        result = list(fts.split(test_data))
        assert len(result) == n_splits
        for i, isplit in enumerate(result):
            assert len(isplit) == 2
            assert len(isplit[0]) == len(test_data) - (n_splits - i) * horizon
            assert len(isplit[1]) == horizon
            assert np.array_equal(isplit[0], np.arange(len(test_data) - (n_splits - i) * horizon))
            assert np.array_equal(
                isplit[1],
                np.arange(horizon) + len(test_data) - (n_splits - i) * horizon,
            )
    else:
        with pytest.raises(expected_error):
            _ = list(fts.split(test_data))


text:
# coding: utf-8

# In[87]:

#基于分词的文本相似度的计算,
#利用jieba分词进行中文分析
import jieba
import jieba.posseg as pseg
from jieba import analyse
import numpy as np
import os

'''
文本相似度的计算，基于几种常见的算法的实现
'''
class TextSimilarity(object):

    def __init__(self,file_a,file_b):
        '''
        初始化类行
        '''
        str_a = ''
        str_b = ''
        if not os.path.isfile(file_a):
            print(file_a,"is not file")
            return
        elif not os.path.isfile(file_b):
            print(file_b,"is not file")
            return
        else:
            with open(file_a,'r') as f:
                for line in f.readlines():
                    str_a += line.strip()

                f.close()
            with open(file_b,'r') as f:
                for line in f.readlines():
                    str_b += line.strip()

                f.close()

        self.str_a = str_a
        self.str_b = str_b

    #get LCS(longest common subsquence),DP
    def lcs(self,str_a, str_b):
        lensum = float(len(str_a) + len(str_b))
        #得到一个二维的数组，类似用dp[lena+1][lenb+1],并且初始化为0
        lengths = [[0 for j in range(len(str_b)+1)] for i in range(len(str_a)+1)]

        #enumerate(a)函数： 得到下标i和a[i]
        for i, x in enumerate(str_a):
            for j, y in enumerate(str_b):
                if x == y:
                    lengths[i+1][j+1] = lengths[i][j] + 1
                else:
                    lengths[i+1][j+1] = max(lengths[i+1][j], lengths[i][j+1])

        #到这里已经得到最长的子序列的长度，下面从这个矩阵中就是得到最长子序列
        result = ""
        x, y = len(str_a), len(str_b)
        while x != 0 and y != 0:
            #证明最后一个字符肯定没有用到
            if lengths[x][y] == lengths[x-1][y]:
                x -= 1
            elif lengths[x][y] == lengths[x][y-1]:
                y -= 1
            else: #用到的从后向前的当前一个字符
                assert str_a[x-1] == str_b[y-1] #后面语句为真，类似于if(a[x-1]==b[y-1]),执行后条件下的语句
                result = str_a[x-1] + result #注意这一句，这是一个从后向前的过程
                x -= 1
                y -= 1

                #和上面的代码类似
                #if str_a[x-1] == str_b[y-1]:
                #    result = str_a[x-1] + result #注意这一句，这是一个从后向前的过程
                #    x -= 1
                #    y -= 1
        longestdist = lengths[len(str_a)][len(str_b)]
        ratio = longestdist/min(len(str_a),len(str_b))
        #return {'longestdistance':longestdist, 'ratio':ratio, 'result':result}
        return ratio

    def minimumEditDistance(self,str_a,str_b):
        '''
        最小编辑距离，只有三种操作方式 替换、插入、删除
        '''
        lensum = float(len(str_a) + len(str_b))
        if len(str_a) > len(str_b): #得到最短长度的字符串
            str_a,str_b = str_b,str_a
        distances = range(len(str_a) + 1) #设置默认值
        for index2,char2 in enumerate(str_b): #str_b > str_a
            newDistances = [index2+1] #设置新的距离，用来标记
            for index1,char1 in enumerate(str_a):
                if char1 == char2: #如果相等，证明在下标index1出不用进行操作变换，最小距离跟前一个保持不变，
                    newDistances.append(distances[index1])
                else: #得到最小的变化数，
                    newDistances.append(1 + min((distances[index1],   #删除
                                                 distances[index1+1], #插入
                                                 newDistances[-1])))  #变换
            distances = newDistances #更新最小编辑距离

        mindist = distances[-1]
        ratio = (lensum - mindist)/lensum
        #return {'distance':mindist, 'ratio':ratio}
        return ratio

    def levenshteinDistance(self,str1, str2):
        '''
        编辑距离——莱文斯坦距离,计算文本的相似度
        '''
        m = len(str1)
        n = len(str2)
        lensum = float(m + n)
        d = []
        for i in range(m+1):
            d.append([i])
        del d[0][0]
        for j in range(n+1):
            d[0].append(j)
        for j in range(1,n+1):
            for i in range(1,m+1):
                if str1[i-1] == str2[j-1]:
                    d[i].insert(j,d[i-1][j-1])
                else:
                    minimum = min(d[i-1][j]+1, d[i][j-1]+1, d[i-1][j-1]+2)
                    d[i].insert(j, minimum)
        ldist = d[-1][-1]
        ratio = (lensum - ldist)/lensum
        #return {'distance':ldist, 'ratio':ratio}
        return ratio

    @classmethod
    def splitWords(self,str_a):
        '''
        接受一个字符串作为参数，返回分词后的结果字符串(空格隔开)和集合类型
        '''
        wordsa=pseg.cut(str_a)
        cuta = ""
        seta = set()
        for key in wordsa:
            #print(key.word,key.flag)
            cuta += key.word + " "
            seta.add(key.word)

        return [cuta, seta]

    def JaccardSim(self,str_a,str_b):
        '''
        Jaccard相似性系数
        计算sa和sb的相似度 len（sa & sb）/ len（sa | sb）
        '''
        seta = self.splitWords(str_a)[1]
        setb = self.splitWords(str_b)[1]

        sa_sb = 1.0 * len(seta & setb) / len(seta | setb)

        return sa_sb

    def countIDF(self,text,topK):
        '''
        text:字符串，topK根据TF-IDF得到前topk个关键词的词频，用于计算相似度
        return 词频vector
        '''
        tfidf = analyse.extract_tags

        cipin = {} #统计分词后的词频

        fenci = jieba.cut(text)

        #记录每个词频的频率
        for word in fenci:
            if word not in cipin.keys():
                cipin[word] = 0
            cipin[word] += 1

        # 基于tfidf算法抽取前10个关键词，包含每个词项的权重
        keywords = tfidf(text,topK,withWeight=True)

        ans = []
        # keywords.count(keyword)得到keyword的词频
        # help(tfidf)
        # 输出抽取出的关键词
        for keyword in keywords:
            #print(keyword ," ",cipin[keyword[0]])
            ans.append(cipin[keyword[0]]) #得到前topk频繁词项的词频

        return ans
    @staticmethod
    def cos_sim(a,b):
        a = np.array(a)
        b = np.array(b)

        #return {"文本的余弦相似度:":np.sum(a*b) / (np.sqrt(np.sum(a ** 2)) * np.sqrt(np.sum(b ** 2)))}
        return np.sum(a*b) / (np.sqrt(np.sum(a ** 2)) * np.sqrt(np.sum(b ** 2)))
    @staticmethod
    def eucl_sim(a,b):
        a = np.array(a)
        b = np.array(b)
        #print(a,b)
        #print(np.sqrt((np.sum(a-b)**2)))
        #return {"文本的欧几里德相似度:":1/(1+np.sqrt((np.sum(a-b)**2)))}
        return 1/(1+np.sqrt((np.sum(a-b)**2)))
    @staticmethod
    def pers_sim(a,b):
        a = np.array(a)
        b = np.array(b)

        a = a - np.average(a)
        b = b - np.average(b)

        #print(a,b)
        #return {"文本的皮尔森相似度:":np.sum(a*b) / (np.sqrt(np.sum(a ** 2)) * np.sqrt(np.sum(b ** 2)))}
        return np.sum(a*b) / (np.sqrt(np.sum(a ** 2)) * np.sqrt(np.sum(b ** 2)))

    def splitWordSimlaryty(self,str_a,str_b,topK = 20,sim =cos_sim):
        '''
        基于分词求相似度，默认使用cos_sim 余弦相似度,默认使用前20个最频繁词项进行计算
        '''
        #得到前topK个最频繁词项的字频向量
        vec_a = self.countIDF(str_a,topK)
        vec_b = self.countIDF(str_b,topK)

        return sim(vec_a,vec_b)

    @staticmethod
    def string_hash(self,source):  #局部哈希算法的实现
        if source == "":
            return 0
        else:
            #ord()函数 return 字符的Unicode数值
            x = ord(source[0]) << 7
            m = 1000003  #设置一个大的素数
            mask = 2 ** 128 - 1  #key值
            for c in source:  #对每一个字符基于前面计算hash
                x = ((x * m) ^ ord(c)) & mask

            x ^= len(source) #
            if x == -1:  #证明超过精度
                x = -2
            x = bin(x).replace('0b', '').zfill(64)[-64:]
            #print(source,x)

        return str(x)

    def simhash(self,str_a,str_b):
        '''
        使用simhash计算相似度
        '''
        pass

max_stars_repo_name: xiaorancs/text-similarity

max_issues_repo_name: xiaorancs/text-similarity

max_forks_repo_name: xiaorancs/text-similarity


text:
import numpy as np
import pandas as pd
import sys

def validate(file_name):
    df = pd.read_csv(file_name)
    df_correct = pd.read_csv('data.csv')
    try:
        if (np.any(df_correct.dtypes != df.dtypes)):
            raise ValueError()
    except (ValueError):
        print('Input DataFrame does not contain the correct data type.')
        exit()
    try:
        if (sorted(list(df)) != sorted(list(df_correct))):
            raise ValueError()
    except (ValueError):
        print('Input DataFrame does not contain the expected columns.')
        exit()
    try:
        if len(df.index) < 1:
            raise ValueError()
    except (ValueError):
        print("Input DataFrame does not have at least 1 row.")
        exit()
    print('Congratulations, input dataframe passes all checks.')

if __name__ == '__main__':
    validate(sys.argv[1])


text:
"""Unit tests for :class:`esmf_regrid.esmf_regridder.Regridder`."""

import ESMF
import numpy as np
from numpy import ma

from esmf_regrid.esmf_regridder import GridInfo, Regridder
from esmf_regrid.tests import make_grid_args

def test_esmpy_normalisation():
    """
    Integration test for :meth:`~esmf_regrid.esmf_regridder.Regridder`.

    Checks against ESMF to ensure results are consistent.
    """
    src_data = np.array(
        [
            [1.0, 1.0],
            [1.0, 0.0],
            [1.0, 0.0],
        ],
    )
    src_mask = np.array(
        [
            [True, False],
            [False, False],
            [False, False],
        ]
    )
    src_array = ma.array(src_data, mask=src_mask)

    lon, lat, lon_bounds, lat_bounds = make_grid_args(2, 3)
    src_grid = GridInfo(lon, lat, lon_bounds, lat_bounds)
    src_esmpy_grid = src_grid._make_esmf_grid()
    src_esmpy_grid.add_item(ESMF.GridItem.MASK, staggerloc=ESMF.StaggerLoc.CENTER)
    src_esmpy_grid.mask[0][...] = src_mask
    src_field = ESMF.Field(src_esmpy_grid)
    src_field.data[...] = src_data

    lon, lat, lon_bounds, lat_bounds = make_grid_args(3, 2)
    tgt_grid = GridInfo(lon, lat, lon_bounds, lat_bounds)
    tgt_field = tgt_grid.make_esmf_field()

    regridder = Regridder(src_grid, tgt_grid)

    regridding_kwargs = {
        "ignore_degenerate": True,
        "regrid_method": ESMF.RegridMethod.CONSERVE,
        "unmapped_action": ESMF.UnmappedAction.IGNORE,
        "factors": True,
        "src_mask_values": [1],
    }
    esmpy_fracarea_regridder = ESMF.Regrid(
        src_field, tgt_field, norm_type=ESMF.NormType.FRACAREA, **regridding_kwargs
    )
    esmpy_dstarea_regridder = ESMF.Regrid(
        src_field, tgt_field, norm_type=ESMF.NormType.DSTAREA, **regridding_kwargs
    )

    tgt_field_dstarea = esmpy_dstarea_regridder(src_field, tgt_field)
    result_esmpy_dstarea = tgt_field_dstarea.data
    result_dstarea = regridder.regrid(src_array, norm_type="dstarea")
    assert ma.allclose(result_esmpy_dstarea, result_dstarea)

    tgt_field_fracarea = esmpy_fracarea_regridder(src_field, tgt_field)
    result_esmpy_fracarea = tgt_field_fracarea.data
    result_fracarea = regridder.regrid(src_array, norm_type="fracarea")
    assert ma.allclose(result_esmpy_fracarea, result_fracarea)

max_stars_repo_name: trexfeathers/iris-esmf-regrid

max_issues_repo_name: trexfeathers/iris-esmf-regrid

max_forks_repo_name: trexfeathers/iris-esmf-regrid

reason: import numpy,from numpy


text:
import glob
import pickle

import numpy as np
from dataloader.video import Video
import os
import natsort
from tqdm import tqdm

audio_path = r'E:\Datasets\ABAW2\audio'
video_path = r'E:\Datasets\ABAW2\video/all'
crop_alined_path = r'E:\Datasets\ABAW2\cropped_aligned'
prediction_path = r'E:\PyProjects\compete\ABAW2020TNT\result2'
new_dir = 'prediction_new3'
videos = os.listdir(video_path)

# video_dict = dict()
# for v in tqdm(videos):
#     if not v.endswith('.mp4') and not v.endswith('.avi'):
#         continue
#     video_file = os.path.join(video_path, v)
#     video = Video(video_file)
#     n_frame = int(video.num_frames)
#     video_dict.update({v:n_frame})
# print(video_dict)
# print(len(video_dict))
# pickle.dump(video_dict, open('n_video_frames.pkl', 'wb'))
video_dict = pickle.load(open('n_video_frames.pkl', 'rb'))
def nearest_interp(source_list, target_len):
    L = len(source_list)
    if L > target_len:
        print(f'L：{L}, target_len：{target_len}')
        print(source_list)
    source_list = sorted(source_list)
    if target_len <= len(source_list):
        return list(range(L))
    out = []
    index = 0
    for i in range(target_len):
        try:
            for _ in range(source_list[index + 1] - source_list[index]):
                out.append(index)
            index += 1
        except:
            for _ in range(target_len - len(out)):
                out.append(index)

    return out

for task in ['AU', 'EXPR', 'VA']:
    prediction_files = glob.glob(os.path.join(prediction_path, task) + '/*.txt')
    for pf in tqdm(prediction_files):
        basename = os.path.basename(pf)
        filename = basename.split('.')[0]
        aligned_name = filename
        dirname = os.path.dirname(pf)
        filename = filename.replace('_main', '').replace('_left', '').replace('_right', '')
        if filename + '.mp4' in videos:
            filename += '.mp4'
        elif filename + '.avi' in videos:
            filename += '.avi'
        else:
            raise Exception(f'No such file:{filename}')
        # video_file = os.path.join(video_path, filename)
        # video = Video(video_file)
        # n_frame = int(video.num_frames)
        n_frame = video_dict[filename]
        # n_frame = video.count_frames()
        # video.release()
        #
        # if n_frame2 != n_frame:
        #     print(filename, n_frame, n_frame2)
        crop_alined_dir = os.path.join(crop_alined_path, aligned_name)
        frames = natsort.natsorted(os.listdir(crop_alined_dir))
        # print(crop_alined_dir)
        frames = [int(frame.split('.')[0]) for frame in frames if '.jpg' in frame]
        with open(pf, 'r') as f:
            pred = f.readlines()
        os.makedirs(f'{new_dir}/{task}', exist_ok=True)
        with open(f'{new_dir}/{task}/{basename}', 'w') as new_f:
            indices = nearest_interp(source_list=frames, target_len=n_frame)
            new_f.write(pred[0])
            assert len(frames) == len(pred) - 1
            for i in range(n_frame):
                index = indices[i] + 1
                p = pred[index]
                new_f.write(p)
            print(n_frame, index, frames[index - 1], len(indices))

# x = [1, 2, 4, 5]
# a = nearest_interp(x, target_len=5)
# print(a)
# print(len(a))
# c = [x[i] for i in a]
# print(c)


text:
import numpy as np
from union_find import union, find, connected

def get_surroundings(matrix, coord):
    """
    Get surrounding coordinates only if their indexes are part of the matrix
    """
    width = np.shape(matrix)[0]
    height = np.shape(matrix)[1]
    coordinates = []

    # top
    (
        coordinates.append((coord[0], coord[1] - 1))
        if coord[1] - 1 >= 0
        else None
    )
    # bottom
    (
        coordinates.append((coord[0], coord[1] + 1))
        if coord[1] + 1 < height
        else None
    )
    # left
    (
        coordinates.append((coord[0] - 1, coord[1]))
        if coord[0] - 1 >= 0
        else None
    )
    # right
    (
        coordinates.append((coord[0] + 1, coord[1]))
        if coord[0] + 1 < width
        else None
    )

    return coordinates

def find_indexes_of_cords(coordinates, hashTable):
    indexes = []
    for item in coordinates:
        indexes.append(hashTable.index(item))
    return indexes

def get_possible_next_steps(matrix, hashTable, coordinate):
    """
    Get possible next steps indexes for the maze route.
    """
    surroundings_coordinates = get_surroundings(matrix, coordinate)
    # get only surrounding coordinates that represent an "EMPTY" /not "WALL"
    filtered_surroundings = []
    for coord in surroundings_coordinates:
        item = matrix[coord[0]][coord[1]]
        if item == 0:
            filtered_surroundings.append(coord)

    indexes = find_indexes_of_cords(filtered_surroundings, hashTable)
    return indexes

def get_non_connected_next_steps(data, currentStep, possible_next_steps):
    next_steps = []
    for step in possible_next_steps:
        if not connected(data, currentStep, step):
            next_steps.append(step)
    return next_steps

max_stars_repo_name: monkey-cli/monkey-playground

max_issues_repo_name: monkey-cli/monkey-playground

max_forks_repo_name: monkey-cli/monkey-playground


text:
"""Trainining script for seq2seq text-to-speech synthesis model.

usage: train.py [options]

options:
    --data-root=<dir>            Directory contains preprocessed features.
    --checkpoint-dir=<dir>       Directory where to save model checkpoints [default: checkpoints].
    --hparams=<parmas>           Hyper parameters [default: ].
    --preset=<json>              Path of preset parameters (json).
    --checkpoint=<path>          Restore model from checkpoint path if given.
    --checkpoint-seq2seq=<path>  Restore seq2seq model from checkpoint path.
    --checkpoint-postnet=<path>  Restore postnet model from checkpoint path.
    --restore-parts=<path>       Restore part of the model.
    --log-event-path=<name>      Log event path.
    --reset-optimizer            Reset optimizer.
    --load-embedding=<path>      Load embedding from checkpoint.
    --speaker-id=<N>             Use specific speaker of data in case for multi-speaker datasets.
    -h, --help                   Show this help message and exit
"""
from docopt import docopt

import sys
import gc
import platform
from os.path import dirname, join
from tqdm import tqdm, trange
from datetime import datetime

# The deepvoice3 model
from deepvoice3_pytorch import frontend, builder
import audio
import lrschedule

import torch
from torch import nn
from torch import optim
import torch.backends.cudnn as cudnn
from torch.utils import data as data_utils
from torch.utils.data.sampler import Sampler
import numpy as np

from nnmnkwii.datasets import FileSourceDataset, FileDataSource
from os.path import join, expanduser
import random

import librosa.display
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot as plt
import sys
import os
#from tensorboardX import SummaryWriter
from torch.utils.tensorboard import SummaryWriter
from matplotlib import cm
from warnings import warn
from hparams import hparams, hparams_debug_string
import training_module as tm
from training_module import TextDataSource, MelSpecDataSource

global_step = 0
global_epoch = 0
use_cuda = torch.cuda.is_available()
if use_cuda:
    cudnn.benchmark = False

_frontend = None  # to be set later

class PyTorchDataset(object):
    def __init__(self, X, Mel):
        self.X = X
        self.Mel = Mel
        # alias
        self.multi_speaker = X.file_data_source.multi_speaker

    def __getitem__(self, idx):
        if self.multi_speaker:
            text, speaker_id = self.X[idx]
            return text, self.Mel[idx], speaker_id
        else:
            return self.X[idx], self.Mel[idx]

    def __len__(self):
        return len(self.X)

def collate_fn(batch):
    """Create batch"""
    r = hparams.outputs_per_step
    multi_speaker = len(batch[0]) == 3

    # Lengths
    input_lengths = [len(x[0]) for x in batch]
    max_input_len = max(input_lengths)

    target_lengths = [len(x[1]) for x in batch]
    max_target_len = max(target_lengths)

    if max_input_len % r != 0:
        max_input_len += r - max_input_len % r
        assert max_input_len % r == 0

    if max_target_len % r != 0:
        max_target_len += r - max_target_len % r
        assert max_target_len % r == 0

    # Set 0 for zero beginning padding
    # imitates initial decoder states
    b_pad = r
    max_target_len += b_pad

    a = np.array([tm._pad(x[0], max_input_len) for x in batch], dtype=np.int)
    x_batch = torch.LongTensor(a)

    input_lengths = torch.LongTensor(input_lengths)
    target_lengths = torch.LongTensor(target_lengths)

    b = np.array([tm._pad_2d(x[1], max_target_len, b_pad=b_pad) for x in batch],
                 dtype=np.float32)
    mel_batch = torch.FloatTensor(b)

    # text positions
    text_positions = np.array([tm._pad(np.arange(1, len(x[0]) + 1), max_input_len)
                               for x in batch], dtype=np.int)
    text_positions = torch.LongTensor(text_positions)

    max_decoder_target_len = max_target_len // r

    # frame positions
    s, e = 1, max_decoder_target_len + 1
    # if b_pad > 0:
    #    s, e = s - 1, e - 1
    frame_positions = torch.arange(s, e).long().unsqueeze(0).expand(
        len(batch), max_decoder_target_len).clone()

    # done flags
    done = np.array([tm._pad(np.zeros(len(x[1]) // r - 1),
                          max_decoder_target_len, constant_values=1)
                     for x in batch])
    done = torch.FloatTensor(done).unsqueeze(-1)

    if multi_speaker:
        speaker_ids = torch.LongTensor([x[2] for x in batch])
    else:
        speaker_ids = None
    return x_batch, input_lengths, mel_batch, \
        (text_positions, frame_positions), done, target_lengths, speaker_ids

#TODO: Neural Vocoderでテストできるようにする
def eval_model(global_step, writer, device, model, checkpoint_dir, ismultispeaker):
    # harded coded
    texts = [
        "And debtors might practically have as much as they liked%if they could only pay for it.",
        "There's a way to measure the acute emotional intelligence that has never gone out of style.",
        "President trump met with other leaders at the group of 20 conference.",
        "Generative adversarial network or variational auto encoder.",
        "Please call stella.",
        "Some have accepted this as a miracle without any physical explanation.",
    ]
    import synthesis
    synthesis._frontend = _frontend

    eval_output_dir = join(checkpoint_dir, "eval")
    os.makedirs(eval_output_dir, exist_ok=True)

    # Prepare model for evaluation
    model_eval = tm.build_model().to(device)
    model_eval.load_state_dict(model.state_dict())

    # hard coded
    speaker_ids = [0, 1, 10] if ismultispeaker else [None]
    for speaker_id in speaker_ids:
        speaker_str = "multispeaker{}".format(speaker_id) if speaker_id is not None else "single"

        for idx, text in enumerate(texts, 1):
            model_eval.eval()
            model_eval.make_generation_fast_()

            sequence = np.array(_frontend.text_to_sequence(text, p=0.5))
            #import pdb; pdb.set_trace()
            sequence = torch.from_numpy(sequence).unsqueeze(0).long().to(device)
            text_positions = torch.arange(1, sequence.size(-1) + 1).unsqueeze(0).long().to(device)
            speaker_ids = None if speaker_id is None else torch.LongTensor([speaker_id]).to(device)

            # Greedy decoding
            with torch.no_grad():
                mel, alignments, done = model_eval(
                    sequence, text_positions=text_positions, speaker_ids=speaker_ids)
            alignments = alignments[0].cpu().data.numpy()
            mel = mel[0].cpu().data.numpy()
            mel = audio._denormalize(mel)

            # Alignment
            for i, alignment in enumerate(alignments, 1):
                alignment_dir = join(eval_output_dir, "alignment_layer{}".format(i))
                os.makedirs(alignment_dir, exist_ok=True)
                path = join(alignment_dir, "step{:09d}_text{}_{}_layer{}_alignment.png".format(
                    global_step, idx, speaker_str, i))
                tm.save_alignment(path, alignment, global_step)
                tag = "eval_text_{}_alignment_layer{}_{}".format(idx, i, speaker_str)
                writer.add_image(tag, np.uint8(cm.viridis(np.flip(alignment, 1)) * 255).T, global_step)

            # Mel
            writer.add_image("(Eval) Predicted mel spectrogram text{}_{}".format(idx, speaker_str),
                             tm.prepare_spec_image(mel).transpose(2,0,1), global_step)

def train(device, model, data_loader, optimizer, writer,
          init_lr=0.002,
          checkpoint_dir=None, checkpoint_interval=None, nepochs=None,
          max_clip=100,
          clip_thresh=1.0):
    r = hparams.outputs_per_step
    current_lr = init_lr

    binary_criterion = nn.BCELoss()
    l1 = nn.L1Loss()

    #save用にNoneデータを準備
    converter_outputs, y = None, None

    global global_step, global_epoch
    while global_epoch < nepochs:
        running_loss = 0.
        print("{}epoch:".format(global_epoch))
        for step, (x, input_lengths, mel, positions, done, target_lengths,
                   speaker_ids) \
                in tqdm(enumerate(data_loader)):
            model.train()
            ismultispeaker = speaker_ids is not None
            # Learning rate schedule
            if hparams.lr_schedule is not None:
                lr_schedule_f = getattr(lrschedule, hparams.lr_schedule)
                current_lr = lr_schedule_f(
                    init_lr, global_step, **hparams.lr_schedule_kwargs)
                for param_group in optimizer.param_groups:
                    param_group['lr'] = current_lr
            optimizer.zero_grad()

            # Used for Position encoding
            text_positions, frame_positions = positions

            # Lengths
            input_lengths = input_lengths.long().numpy()
            decoder_lengths = target_lengths.long().numpy() // r

            max_seq_len = max(input_lengths.max(), decoder_lengths.max())
            if max_seq_len >= hparams.max_positions:
                raise RuntimeError(
                    """max_seq_len ({}) >= max_posision ({})
Input text or decoder targget length exceeded the maximum length.
Please set a larger value for ``max_position`` in hyper parameters.""".format(
                        max_seq_len, hparams.max_positions))

            # Transform data to CUDA device
            x = x.to(device)
            text_positions = text_positions.to(device)
            frame_positions = frame_positions.to(device)
            mel, done = mel.to(device), done.to(device)
            target_lengths = target_lengths.to(device)
            speaker_ids = speaker_ids.to(device) if ismultispeaker else None

            # model output
            mel_outputs, attn, done_hat = model(
                x, mel, speaker_ids=speaker_ids,
                text_positions=text_positions, frame_positions=frame_positions,
                input_lengths=input_lengths)
            # reshape
            mel_outputs = mel_outputs.view(len(mel), -1, mel.size(-1))

            # Losses
            mel_loss = l1(mel_outputs[:, :-r, :], mel[:, r:, :])
            # done:
            done_loss = binary_criterion(done_hat, done)
            #combine Losses
            loss = mel_loss + done_loss

            if global_epoch == 0 and global_step == 0:
                tm.save_states(
                    global_step, writer, mel_outputs, converter_outputs, attn,
                    mel, y, input_lengths, checkpoint_dir)
                tm.save_checkpoint(
                    model, optimizer, global_step, checkpoint_dir, global_epoch)

            # Update
            loss.backward()
            if clip_thresh > 0:
                grad_norm = torch.nn.utils.clip_grad_norm_(
                    model.get_trainable_parameters(), max_clip)
                grad_value = torch.nn.utils.clip_grad_value_(
                    model.get_trainable_parameters(),clip_thresh)
            optimizer.step()

            # Logs
            writer.add_scalar("loss", float(loss.item()), global_step)
            writer.add_scalar("done_loss", float(done_loss.item()), global_step)
            writer.add_scalar("mel_l1_loss", float(mel_loss.item()), global_step)
            if clip_thresh > 0:
                writer.add_scalar("gradient norm", grad_norm, global_step)
            writer.add_scalar("learning rate", current_lr, global_step)

            global_step += 1
            running_loss += loss.item()

            if global_step > 0 and global_step % checkpoint_interval == 0:
                tm.save_states(
                    global_step, writer, mel_outputs, converter_outputs, attn,
                    mel, y, input_lengths, checkpoint_dir)
                tm.save_checkpoint(
                    model, optimizer, global_step, checkpoint_dir, global_epoch)

            if global_step > 1e5 and global_step % hparams.eval_interval == 0 :
                eval_model(global_step, writer, device, model, checkpoint_dir, ismultispeaker)

        averaged_loss = running_loss / (len(data_loader))
        writer.add_scalar("loss (per epoch)", averaged_loss, global_epoch)
        print("Loss: {}".format(running_loss / (len(data_loader))))

        global_epoch += 1

if __name__ == "__main__":
    args = docopt(__doc__)
    print("Command line args:\n", args)
    checkpoint_dir = args["--checkpoint-dir"]
    checkpoint_path = args["--checkpoint"]
    checkpoint_seq2seq_path = args["--checkpoint-seq2seq"]
    checkpoint_postnet_path = args["--checkpoint-postnet"]
    load_embedding = args["--load-embedding"]
    checkpoint_restore_parts = args["--restore-parts"]
    speaker_id = args["--speaker-id"]
    speaker_id = int(speaker_id) if speaker_id is not None else None
    preset = args["--preset"]

    data_root = args["--data-root"]
    if data_root is None:
        data_root = join(dirname(__file__), "data", "ljspeech")

    log_event_path = args["--log-event-path"]
    reset_optimizer = args["--reset-optimizer"]

    # Load preset if specified
    if preset is not None:
        with open(preset) as f:
            hparams.parse_json(f.read())

    # Override hyper parameters
    hparams.parse(args["--hparams"])

    # Preventing Windows specific error such as MemoryError
    # Also reduces the occurrence of THAllocator.c 0x05 error in Widows build of PyTorch
    if platform.system() == "Windows":
        print(" [!] Windows Detected - IF THAllocator.c 0x05 error occurs SET num_workers to 1")

    assert hparams.name == "deepvoice3"
    print(hparams_debug_string())

    _frontend = getattr(frontend, hparams.frontend)

    os.makedirs(checkpoint_dir, exist_ok=True)

    # Input dataset definitions
    X = FileSourceDataset(TextDataSource(data_root, speaker_id))
    Mel = FileSourceDataset(MelSpecDataSource(data_root, speaker_id))

    # Prepare sampler
    frame_lengths = Mel.file_data_source.frame_lengths
    sampler = tm.PartialyRandomizedSimilarTimeLengthSampler(
        frame_lengths, batch_size=hparams.batch_size)

    # Dataset and Dataloader setup
    dataset = PyTorchDataset(X, Mel)
    data_loader = data_utils.DataLoader(
        dataset, batch_size=hparams.batch_size,
        num_workers=hparams.num_workers, sampler=sampler,
        collate_fn=collate_fn, pin_memory=hparams.pin_memory)

    device = torch.device("cuda" if use_cuda else "cpu")

    # Model
    model = tm.build_model().to(device)

    optimizer = optim.Adam(model.get_trainable_parameters(),
                           lr=hparams.initial_learning_rate, betas=(
        hparams.adam_beta1, hparams.adam_beta2),
        eps=hparams.adam_eps, weight_decay=hparams.weight_decay,
        amsgrad=hparams.amsgrad)

    if checkpoint_restore_parts is not None:
        tm.restore_parts(checkpoint_restore_parts, model)

    # Load checkpoints
    if checkpoint_path is not None:
        model, global_step, global_epoch = tm.load_checkpoint(checkpoint_path, model, optimizer, reset_optimizer)

    # Load embedding
    if load_embedding is not None:
        print("Loading embedding from {}".format(load_embedding))
        tm._load_embedding(load_embedding, model)

    # Setup summary writer for tensorboard
    if log_event_path is None:
        if platform.system() == "Windows":
            log_event_path = "log/run-test" + \
                str(datetime.now()).replace(" ", "_").replace(":", "_")
        else:
            log_event_path = "log/run-test" + str(datetime.now()).replace(" ", "_")
    print("Log event path: {}".format(log_event_path))
    writer = SummaryWriter(log_event_path)

    # Train!
    try:
        train(device, model, data_loader, optimizer, writer,
              init_lr=hparams.initial_learning_rate,
              checkpoint_dir=checkpoint_dir,
              checkpoint_interval=hparams.checkpoint_interval,
              nepochs=hparams.nepochs,
              max_clip=hparams.max_clip,
              clip_thresh=hparams.clip_thresh)
    except KeyboardInterrupt:
        tm.save_checkpoint(
            model, optimizer, global_step, checkpoint_dir, global_epoch)

    print("Finished")
    sys.exit(0)


text:
import os
import sys
import time
import logging

import numpy as np

import torch
from torch import optim
from torch.optim import lr_scheduler
from torch.utils.data import DataLoader
import torchvision.models as models
import torchvision.datasets as datasets

from models.siamese_net import SiameseNetwork
from models.l2_contrastive_loss import L2ContrastiveLoss
from utils.trainer import Trainer
from utils.helper import Save_Handle, AverageMeter, worker_init_fn
from utils.visualizer import ImageDisplayer, EmbeddingDisplayer
from datasets.spatial import SpatialDataset
from datasets.cifar10 import PosNegCifar10

class CoTrainer(Trainer):
    def setup(self):
        """initialize the datasets, model, loss and optimizer"""
        args = self.args
        self.vis = ImageDisplayer(args, self.save_dir)
        self.emb = EmbeddingDisplayer(args, self.save_dir)

        if torch.cuda.is_available():
            self.device = torch.device("cuda")
            self.device_count = torch.cuda.device_count()
            logging.info('using {} gpus'.format(self.device_count))
        else:
            raise Exception("gpu is not available")

        if args.cifar10:
            # Download and create datasets
            or_train = datasets.CIFAR10(root="CIFAR10_Dataset", train=True, transform=None, download=True)
            or_val = datasets.CIFAR10(root="CIFAR10_Dataset", train=False, transform=None, download=True)

            # splits CIFAR10 into two streams
            self.datasets = {x: PosNegCifar10((or_train if x == 'train' else or_val),
                                              phase=x) for x in ['train', 'val']}
        else:
            self.datasets = {x: SpatialDataset(os.path.join(args.data_dir, x),
                                            args.crop_size,
                                            args.div_num,
                                            args.aug) for x in ['train', 'val']}

        self.dataloaders = {x: DataLoader(self.datasets[x],
                                        batch_size=args.batch_size,
                                        shuffle=(True if x == 'train' else False),
                                        num_workers=args.num_workers*self.device_count,
                                        pin_memory=(True if x == 'train' else False),
                                        worker_init_fn=worker_init_fn) for x in ['train', 'val']}

        # Define model, loss, optim
        self.model = SiameseNetwork(models.__dict__[args.arch], pattern_feature = args.pattern_feature)
        self.model.to(self.device)

        self.criterion = L2ContrastiveLoss(args.margin)
        self.criterion.to(self.device)

        self.optimizer = optim.SGD(self.model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay)

        self.scheduler = lr_scheduler.MultiStepLR(self.optimizer, milestones=[80, 120, 160, 200, 250], gamma=0.1)

        self.start_epoch = 0
        self.best_loss = np.inf
        if args.resume:
            suf = args.resume.rsplit('.', 1)[-1]
            if suf == 'tar':
                checkpoint = torch.load(args.resume, self.device)
                self.model.load_state_dict(checkpoint['model_state_dict'])
                self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
                self.start_epoch = checkpoint['epoch'] + 1
            elif suf == 'pth':
                self.model.load_state_dict(torch.load(args.resume, self.device))

        self.save_list = Save_Handle(max_num=args.max_model_num)

    def train(self):
        """training process"""
        args = self.args
        for epoch in range(self.start_epoch, args.max_epoch):
            logging.info('-'*5 + 'Epoch {}/{}'.format(epoch, args.max_epoch - 1) + '-'*5)
            self.epoch = epoch
            self.train_epoch(epoch)
            if epoch % args.val_epoch == 0 and epoch >= args.val_start:
                self.val_epoch(epoch)

    def train_epoch(self, epoch):
        epoch_loss = AverageMeter()
        epoch_start = time.time()
        self.model.train()  # Set model to training mode

        for step, (input1, input2, target, label) in enumerate(self.dataloaders['train']):
            input1 = input1.to(self.device)
            input2 = input2.to(self.device)
            target = target.to(self.device)

            with torch.set_grad_enabled(True):
                output1, output2 = self.model(input1, input2)
                loss = self.criterion(output1, output2, target)
                epoch_loss.update(loss.item(), input1.size(0))
                self.optimizer.zero_grad()
                loss.backward()
                self.optimizer.step()
                self.scheduler.step()

            # visualize
            if step == 0:
                self.vis(epoch, 'train', input1, input2, target)
                self.emb(output1, label, epoch, 'train')

        logging.info('Epoch {} Train, Loss: {:.5f}, Cost {:.1f} sec'
                     .format(self.epoch, epoch_loss.get_avg(), time.time()-epoch_start))

        model_state_dic = self.model.state_dict()
        save_path = os.path.join(self.save_dir, '{}_ckpt.tar'.format(self.epoch))
        torch.save({
            'epoch': self.epoch,
            'optimizer_state_dict': self.optimizer.state_dict(),
            'model_state_dict': model_state_dic
        }, save_path)
        self.save_list.append(save_path)  # control the number of saved models

    def val_epoch(self, epoch):
        epoch_start = time.time()
        self.model.eval()  # Set model to evaluate mode
        epoch_loss = AverageMeter()

        for step, (input1, input2, target, label) in enumerate(self.dataloaders['val']):
            input1 = input1.to(self.device)
            input2 = input2.to(self.device)
            target = target.to(self.device)
            with torch.set_grad_enabled(False):
                output1, output2 = self.model(input1, input2)
                loss = self.criterion(output1, output2, target)
                epoch_loss.update(loss.item(), input1.size(0))

            # visualize
            if step == 0:
                self.vis(epoch, 'val', input1, input2, target)
                self.emb(output1, label, epoch, 'val')

        logging.info('Epoch {} Val, Loss: {:.5f}, Cost {:.1f} sec'
                     .format(self.epoch, epoch_loss.get_avg(), time.time()-epoch_start))

        model_state_dic = self.model.state_dict()
        if self.best_loss > epoch_loss.get_avg():
            self.best_loss = epoch_loss.get_avg()
            logging.info("save min loss {:.2f} model epoch {}".format(self.best_loss, self.epoch))
            torch.save(model_state_dic, os.path.join(self.save_dir, 'best_model.pth'))

max_stars_repo_name: junprog/contrastive-baseline

max_issues_repo_name: junprog/contrastive-baseline

max_forks_repo_name: junprog/contrastive-baseline


text:
import morphr as mo

import anurbs as an
import eqlib as eq
import numpy as np

REDUCED_SHELL_3P = mo.ReducedIgaShell

class ApplyReducedShell(mo.Task):
    membrane_stiffness: float
    bending_stiffness: float
    weight: float = 1

    def run(self, config, job, data, log):
        cad_model = data.get('cad_model', None)
        model_tolerance = job.model_tolerance

        nb_objectives = 0

        # FIXME: Check for None

        data['nodes'] = data.get('nodes', {})
        elements = []

        for key, face in cad_model.of_type('BrepFace'):
            surface_geometry_key = surface_geometry = face.surface_geometry.data

            if surface_geometry_key not in data['nodes']:
                nodes = []

                for x, y, z in surface_geometry.poles:
                    nodes.append(eq.Node(x, y, z))
                nodes = np.array(nodes, object)
                data['nodes'][surface_geometry_key] = nodes
            else:
                nodes = data['nodes'][surface_geometry_key]

            for u, v, weight in an.integration_points(face, model_tolerance):
                nonzero_indices, shape_functions = surface_geometry.shape_functions_at(u, v, 2)

                element = REDUCED_SHELL_3P(nodes[nonzero_indices], self.membrane_stiffness, self.bending_stiffness)
                element.add(shape_functions, weight)

                elements.append(element)

                nb_objectives += 1

        data['elements'] = data.get('elements', [])
        data['elements'].append(('ReducedIgaShell', elements, self.weight))

        # output

        log.info(f'{len(elements)} elements with {nb_objectives} new objectives')

max_stars_repo_name: oberbichler/Morphr

max_issues_repo_name: dbaumgaertner/Morphr

max_forks_repo_name: dbaumgaertner/Morphr


text:
import functools
import inspect
import numpy as np
import random

from abc import ABC, abstractmethod
from box import Box

class Param(ABC):
    def __init__(self, capsule, key, default):
        self._capsule = capsule
        self._key = key
        self.default = default

    @abstractmethod
    def get_next_value(self):
        pass

    @abstractmethod
    def serialize_as_nni(self):
        pass

    def serialize(self):
        ret = {}
        for k, v in self.__dict__.items():
            if k and k[0] != '_':
                ret[k] = v
        return ret

    @property
    def var_type(self):
        return 'c'

    def __call__(self, f):
        if inspect.ismethod(f) or f.__name__ == '__init__':
            @functools.wraps(f)
            def wrapper(this, params=None, **kwargs):
                if not self._capsule.initialized:
                    raise RuntimeError('Must initialize capsule before use')
                params = self._process_params(params)
                return f(this, params, **kwargs)
        else:
            @functools.wraps(f)
            def wrapper(params=None, **kwargs):
                if not self._capsule.initialized:
                    raise RuntimeError('Must initialize capsule before use')
                params = self._process_params(params)

                return f(params, **kwargs)

        return wrapper

    def _process_params(self, params):
        _params = params
        if not isinstance(_params, Box):
            params = Box(default_box=True)
        else:
            params = _params
        if _params is not None:
            for ks, v in _params.items():
                ks = ks.split('.')
                cur = params
                for k in ks[:-1]:
                    cur = cur[k]
                cur[ks[-1]] = v
        cur = params
        ks = self._key.split('.')
        for k in ks[:-1]:
            cur = cur[k]
        if ks[-1] not in cur:
            cur[ks[-1]] = self._capsule.params[self._key]
        return params

    @abstractmethod
    def encode_as_numerical(self, value):
        pass

    @abstractmethod
    def encoded_bounds(self):
        pass

    @abstractmethod
    def decode_from_numerical(self, coded):
        pass

    def __repr__(self):
        repr_str = f'<{self.__class__.__name__}'
        for k, v in self.serialize().items():
            repr_str += f' {k}={repr(v)}'
        repr_str += '>'
        return repr_str

def _clip_to_q(val, q, low, high):
    return np.clip(np.round(val / q) * q, low, high)

class ConstParam(Param):
    def __init__(self, capsule, key, default, const):
        super().__init__(capsule, key, default)
        self.const = const

    @property
    def var_type(self):
        return 'u'

    def encode_as_numerical(self, value):
        return 0

    def decode_from_numerical(self, coded):
        return 0

    def encoded_bounds(self):
        return None, None

    def get_next_value(self):
        return self.const

    def serialize_as_nni(self):
        return None

class ChoiceParam(Param):
    def __init__(self, capsule, key, default, choices, prob=None):
        super().__init__(capsule, key, default)
        self.choices = choices
        self.prob = prob

    def get_next_value(self):
        return random.choices(self.choices, weights=self.prob, k=1)[0]

    @property
    def var_type(self):
        return 'u'

    def encode_as_numerical(self, value):
        try:
            return self.choices.index(value)
        except:
            raise ValueError('Cannot encode choice: given', value, 'expect', self.choices)

    def encoded_bounds(self):
        return None, None

    def decode_from_numerical(self, coded):
        rounded = int(_clip_to_q(coded, q=1, low=0, high=len(self.choices) - 1))
        return self.choices[rounded]

    def serialize_as_nni(self):
        return {
            '_type': 'choice',
            '_value': self.choices
        }

class OrderedParam(Param):
    def __init__(self, capsule, key, default, choices, prob=None):
        super().__init__(capsule, key, default)
        self.choices = choices
        self.prob = prob

    def get_next_value(self):
        return random.choices(self.choices, weights=self.prob, k=1)[0]

    @property
    def var_type(self):
        return 'o'

    def encode_as_numerical(self, value):
        return self.choices.index(value)

    def decode_from_numerical(self, coded):
        rounded = int(_clip_to_q(coded, q=1, low=0, high=len(self.choices) - 1))
        return self.choices[rounded]

    def encoded_bounds(self):
        return -0.5, len(self.choices) - 0.5

    def serialize_as_nni(self):
        return {
            '_type': 'choice',
            '_value': self.choices
        }

class RandomIntParam(Param):
    def __init__(self, capsule, key, default, low, high):
        super().__init__(capsule, key, default)
        self.low = low
        self.high = high

    def get_next_value(self):
        return np.random.randint(self.low, self.high)

    def encode_as_numerical(self, value):
        return (value - self.low) / (self.high - self.low)

    def decode_from_numerical(self, coded):
        restored = int(_clip_to_q(coded * (self.high - self.low) + self.low, 1, self.low, self.high))
        return restored

    def encoded_bounds(self):
        return (self.low - 0.5) / (self.high - self.low), (self.high + 0.5) / (self.high - self.low)

    def serialize_as_nni(self):
        return {
            '_type': 'randint',
            '_value': [self.low, self.high]
        }

class UniformParam(Param):
    def __init__(self, capsule, key, default, low, high):
        super().__init__(capsule, key, default)
        self.low = low
        self.high = high

    def get_next_value(self):
        return np.random.uniform(self.low, self.high)

    def encode_as_numerical(self, value):
        return (value - self.low) / (self.high - self.low)

    def decode_from_numerical(self, coded):
        restored = np.clip(coded * (self.high - self.low) + self.low, self.low, self.high)
        return restored

    def encoded_bounds(self):
        return 0., 1.

    def serialize_as_nni(self):
        return {
            '_type': 'uniform',
            '_value': [self.low, self.high]
        }

class QUniformParam(Param):
    def __init__(self, capsule, key, default, low, high, q):
        super().__init__(capsule, key, default)
        self.low = low
        self.high = high
        self.q = q

    def get_next_value(self):
        return _clip_to_q(random.uniform(self.low, self.high), self.q, self.low, self.high)

    def encode_as_numerical(self, value):
        return (value - self.low) / (self.high - self.low)

    def decode_from_numerical(self, coded):
        restored = coded * (self.high - self.low) + self.low
        return _clip_to_q(restored, self.q, self.low, self.high)

    def encoded_bounds(self):
        return 0., 1.

    def serialize_as_nni(self):
        return {
            '_type': 'quniform',
            '_value': [self.low, self.high, self.q]
        }

class LogUniformParam(Param):
    def __init__(self, capsule, key, default, low, high):
        super().__init__(capsule, key, default)
        self.low = low
        self.high = high

    def get_next_value(self):
        return np.exp(np.random.uniform(np.log(self.low), np.log(self.high)))

    def encode_as_numerical(self, value):
        return (np.log(value) - np.log(self.low)) / (np.log(self.high) - np.log(self.low))

    def decode_from_numerical(self, coded):
        restored = np.exp(coded * (np.log(self.high) - np.log(self.low)) + np.log(self.low))
        return restored

    def encoded_bounds(self):
        return 0., 1.

    def serialize_as_nni(self):
        return {
            '_type': 'loguniform',
            '_value': [self.low, self.high]
        }

class QLogUniformParam(Param):
    def __init__(self, capsule, key, default, low, high, q):
        super().__init__(capsule, key, default)
        self.low = low
        self.high = high
        self.q = q

    def get_next_value(self):
        return _clip_to_q(np.exp(np.random.uniform(np.log(self.low), np.log(self.high))), self.q, self.low, self.high)

    def encode_as_numerical(self, value):
        return (np.log(value) - np.log(self.low)) / (np.log(self.high) - np.log(self.low))

    def decode_from_numerical(self, coded):
        restored = np.exp(coded * (np.log(self.high) - np.log(self.low)) + np.log(self.low))
        return _clip_to_q(restored, self.q, self.low, self.high)

    def encoded_bounds(self):
        return 0., 1.

    def serialize_as_nni(self):
        return {
            '_type': 'qloguniform',
            '_value': [self.low, self.high, self.q]
        }

class NormalParam(Param):
    def __init__(self, capsule, key, default, mean, std):
        super().__init__(capsule, key, default)
        self.mean = mean
        self.std = std

    def get_next_value(self):
        return np.random.normal(self.mean, self.std)

    def encode_as_numerical(self, value):
        return (value - self.mean) / self.std

    def decode_from_numerical(self, coded):
        restored = coded * self.std + self.mean
        return restored

    def encoded_bounds(self):
        return None, None

    def serialize_as_nni(self):
        return {
            '_type': 'normal',
            '_value': [self.mean, self.std]
        }

class QNormalParam(Param):
    def __init__(self, capsule, key, default, mean, std, q):
        super().__init__(capsule, key, default)
        self.mean = mean
        self.std = std
        self.q = q

    def get_next_value(self):
        return np.round(np.random.normal(self.mean, self.std) / self.q) * self.q

    def encode_as_numerical(self, value):
        return (value - self.mean) / self.std

    def decode_from_numerical(self, coded):
        restored = coded * self.std + self.mean
        return np.round(restored / self.q) * self.q

    def encoded_bounds(self):
        return None, None

    def serialize_as_nni(self):
        return {
            '_type': 'qnormal',
            '_value': [self.mean, self.std, self.q]
        }

class LogNormalParam(Param):
    def __init__(self, capsule, key, default, mean, std):
        super().__init__(capsule, key, default)
        self.mean = mean
        self.std = std

    def get_next_value(self):
        return np.exp(np.random.normal(self.mean, self.std))

    def encode_as_numerical(self, value):
        return (np.log(value) - self.mean) / self.std

    def decode_from_numerical(self, coded):
        restored = np.exp(coded * self.std + self.mean)
        return restored

    def encoded_bounds(self):
        return None, None

    def serialize_as_nni(self):
        return {
            '_type': 'lognormal',
            '_value': [self.mean, self.std]
        }

class QLogNormalParam(Param):
    def __init__(self, capsule, key, default, mean, std, q):
        super().__init__(capsule, key, default)
        self.mean = mean
        self.std = std
        self.q = q

    def get_next_value(self):
        return np.round(np.exp(np.random.normal(self.mean, self.std)) / self.q) * self.q

    def encode_as_numerical(self, value):
        return (np.log(value) - self.mean) / self.std

    def decode_from_numerical(self, coded):
        restored = np.exp(coded * self.std + self.mean)
        return np.round(restored / self.q) * self.q

    def encoded_bounds(self):
        return None, None

    def serialize_as_nni(self):
        return {
            '_type': 'qlognormal',
            '_value': [self.mean, self.std, self.q]
        }

class ParamGenerator:
    def __init__(self, capsule):
        self.capsule = capsule

    def const(self, key, default, value=None):
        if value is None:
            value = default
        return self._make_param_wrapper(key, ConstParam(self.capsule, key, default, value))

    def choice(self, key, default, choices=None, prob=None):
        if choices is None:
            choices = default
            default = choices[0]
        return self._make_param_wrapper(key, ChoiceParam(self.capsule, key, default, choices, prob))

    def ordered(self, key, default, choices=None, prob=None):
        if choices is None:
            choices = default
            default = choices[0]
        return self._make_param_wrapper(key, OrderedParam(self.capsule, key, default, choices, prob))

    def randint(self, key, default=None, *, low, high):
        if default is None:
            default = low
        return self._make_param_wrapper(key, RandomIntParam(self.capsule, key, default, low, high))

    def unif(self, key, default=None, *, low, high):
        if default is None:
            default = low
        return self._make_param_wrapper(key, UniformParam(self.capsule, key, default, low, high))

    def qunif(self, key, default=None, *, low, high, q):
        if default is None:
            default = low
        return self._make_param_wrapper(key, QUniformParam(self.capsule, key, default, low, high, q))

    def logunif(self, key, default=None, *, low, high):
        if default is None:
            default = low
        return self._make_param_wrapper(key, LogUniformParam(self.capsule, key, default, low, high))

    def qlogunif(self, key, default=None, *, low, high, q):
        if default is None:
            default = low
        return self._make_param_wrapper(key, QLogUniformParam(self.capsule, key, default, low, high, q))

    def normal(self, key, default=None, *, mean, std):
        if default is None:
            default = mean
        return self._make_param_wrapper(key, NormalParam(self.capsule, key, default, mean, std))

    def qnormal(self, key, default=None, *, mean, std, q):
        if default is None:
            default = mean
        return self._make_param_wrapper(key, QNormalParam(self.capsule, key, default, mean, std, q))

    def lognormal(self, key, default=None, *, mean, std):
        if default is None:
            default = np.exp(mean)
        return self._make_param_wrapper(key, LogNormalParam(self.capsule, key, default, mean, std))

    def qlognormal(self, key, default=None, *, mean, std, q):
        if default is None:
            default = np.exp(mean)
        return self._make_param_wrapper(key, QLogNormalParam(self.capsule, key, default, mean, std, q))

    def _make_param_wrapper(self, key, param_wrapper):
        self.capsule._guard_params()
        if key in self.capsule._param_wrappers:
            raise ValueError('duplicate parameter key', key)
        self.capsule._param_wrappers[key] = param_wrapper
        return param_wrapper

if __name__ == '__main__':
    rv = ChoiceParam(None, 'c', 'a', ['a', 'b', 'c'])
    print(rv.encode_as_numerical('a'))
    print(rv.decode_from_numerical(2.1))
    rv = OrderedParam(None, 'c', 'a', ['a', 'b', 'c'])
    print(rv.encode_as_numerical('a'))
    print(rv.decode_from_numerical(1.9))
    rv = UniformParam(None, 'c', 0, -10, 10)
    print(rv.encode_as_numerical(2.1))
    print(rv.decode_from_numerical(0.605))


text:
import json
import os

import tensorflow as tf
import numpy as np
import soundfile as sf

from util.wrapper import load
from analyzer import read_whole_features, SPEAKERS, pw2wav
from analyzer import Tanhize
from datetime import datetime
from importlib import import_module

args = tf.app.flags.FLAGS
tf.app.flags.DEFINE_string('checkpoint', None, 'root of log dir')
tf.app.flags.DEFINE_string('src', 'SM1', 'source speaker {SM1, SM2}')
tf.app.flags.DEFINE_string('trg', 'TM1', 'target speaker {TM1, TM2, TF1}')
tf.app.flags.DEFINE_string('output_dir', './logdir', 'root of output dir')
tf.app.flags.DEFINE_string('module', 'model.vae', 'Module')
tf.app.flags.DEFINE_string('model', None, 'Model')
tf.app.flags.DEFINE_string('file_pattern', './dataset/bin/Testing Set/{}/*.bin', 'file pattern')

if args.model is None:
    raise ValueError(
        '\n  You MUST specify `model`.' +\
        '\n    Use `python convert.py --help` to see applicable options.'
    )

module = import_module(args.module, package=None)
MODEL = getattr(module, args.model)

FS = 16000

def make_output_wav_name(output_dir, filename):
    basename = str(filename, 'utf8')
    basename = os.path.split(basename)[-1]
    basename = os.path.splitext(basename)[0]
    print('Processing {}'.format(basename))
    return os.path.join(
        output_dir,
        '{}_{}_{}.wav'.format(basename, args.src, args.trg)
    )

def get_default_output(logdir_root):
    STARTED_DATESTRING = datetime.now().strftime('%0m%0d-%0H%0M-%0S-%Y')
    logdir = os.path.join(logdir_root, 'output', STARTED_DATESTRING)
    print('Using default logdir: {}'.format(logdir))
    return logdir

def convert_f0(f0, src, trg):
    mu_s, std_s = np.fromfile(os.path.join('./etc', '{}.npf'.format(src)), np.float32)
    mu_t, std_t = np.fromfile(os.path.join('./etc', '{}.npf'.format(trg)), np.float32)
    lf0 = tf.where(f0 > 1., tf.log(f0), f0)
    lf0 = tf.where(lf0 > 1., (lf0 - mu_s)/std_s * std_t + mu_t, lf0)
    lf0 = tf.where(lf0 > 1., tf.exp(lf0), lf0)
    return lf0

def nh_to_nchw(x):
    with tf.name_scope('NH_to_NCHW'):
        x = tf.expand_dims(x, 1)      # [b, h] => [b, c=1, h]
        return tf.expand_dims(x, -1)  # => [b, c=1, h, w=1]

def main():
    logdir, ckpt = os.path.split(args.checkpoint)
    arch = tf.gfile.Glob(os.path.join(logdir, 'architecture*.json'))[0]  # should only be 1 file
    with open(arch) as fp:
        arch = json.load(fp)

    normalizer = Tanhize(
        xmax=np.fromfile('./etc/xmax.npf'),
        xmin=np.fromfile('./etc/xmin.npf'),
    )

    features = read_whole_features(args.file_pattern.format(args.src))

    x = normalizer.forward_process(features['sp'])
    x = nh_to_nchw(x)
    y_s = features['speaker']
    y_t_id = tf.placeholder(dtype=tf.int64, shape=[1,])
    y_t = y_t_id * tf.ones(shape=[tf.shape(x)[0],], dtype=tf.int64)

    machine = MODEL(arch)
    z = machine.encode(x)
    x_t = machine.decode(z, y_t)  # NOTE: the API yields NHWC format
    x_t = tf.squeeze(x_t)
    x_t = normalizer.backward_process(x_t)

    # For sanity check (validation)
    x_s = machine.decode(z, y_s)
    x_s = tf.squeeze(x_s)
    x_s = normalizer.backward_process(x_s)

    f0_s = features['f0']
    f0_t = convert_f0(f0_s, args.src, args.trg)

    output_dir = get_default_output(args.output_dir)

    saver = tf.train.Saver()
    sv = tf.train.Supervisor(logdir=output_dir)
    with sv.managed_session() as sess:
        load(saver, sess, logdir, ckpt=ckpt)
        while True:
            try:
                feat, f0, sp = sess.run(
                    [features, f0_t, x_t],
                    feed_dict={y_t_id: np.asarray([SPEAKERS.index(args.trg)])}
                )
                feat.update({'sp': sp, 'f0': f0})
                y = pw2wav(feat)
                oFilename = make_output_wav_name(output_dir, feat['filename'])
                sf.write(oFilename, y, FS)
            except:
                break

if __name__ == '__main__':
    os.environ["CUDA_VISIBLE_DEVICES"] = "1"
    main()

max_stars_repo_name: phanxuanphucnd/dsc

max_issues_repo_name: phanxuanphucnd/dsc

max_forks_repo_name: phanxuanphucnd/dsc