text:
import pandas as pd
import numpy as np

table = pd.read_csv('table.csv')
print(table)

previous_faults = 0
total_cost = 0
result = np.empty((0, 2))

for index, row in table.iterrows():
    faults, cost_per_fault, detection_rate = row

    detected_faults = (previous_faults + faults) * detection_rate
    cost = detected_faults * cost_per_fault
    result = np.append(result, np.array([[detected_faults, cost]]), axis=0)

    previous_faults = previous_faults + faults - detected_faults

    total_cost += cost

result_table = pd.DataFrame(result, columns=['detect', 'cost'])
print(result_table)
print('Total cost: {}'.format(total_cost))

ext: py

reason: import numpy


text:
import keyboard
import time
import argparse
import numpy as np
from ddqn_game_model import DDQNTrainer, DDQNSolver
from gym_wrappers import MainGymWrapper
from frame import FRAME_SIZE
from renderer import get_renderer

FRAMES_IN_OBSERVATION = 4
INPUT_SHAPE = (FRAMES_IN_OBSERVATION, FRAME_SIZE, FRAME_SIZE)

class Main:
    def __init__(self):
        game_name, game_mode, model_name, render_mode, total_step_limit, total_run_limit, skip, duel = self._args()
        env = MainGymWrapper(game_name, skip)
        game_model = self._game_model(game_mode, model_name, game_name, env.action_space, duel)
        renderer = get_renderer(render_mode, game_model.ddqn)

        self._main_loop(game_model, env, renderer, total_step_limit, total_run_limit)

    def _main_loop(self, game_model, env, renderer, total_step_limit, total_run_limit):
        run = game_model.initial_run
        total_step = game_model.initial_total_step
        viewer = None

        if run != 0:
            print ("Continue from run " + str(run))

        while True:
            if total_run_limit is not None and run >= total_run_limit:
                print ("Reached total run limit of: " + str(total_run_limit))
                exit(0)

            run += 1
            renderer.reset()
            current_state = env.reset()
            step = 0
            score = 0
            while True:
                if keyboard.is_pressed("q"):
                    print ("Quit button pressed, saving state")
                    game_model.save(run, total_step)
                    exit(0)

                if total_step >= total_step_limit:
                    print ("Reached total step limit of: " + str(total_step_limit))
                    exit(0)
                total_step += 1
                step += 1

                env.render(renderer)

                action = game_model.move(current_state)
                next_state, reward, terminal = env.step(action)
                score += reward
                game_model.remember(current_state, action, reward, next_state, terminal)
                current_state = next_state
                game_model.step_update(total_step)
                if terminal:
                    print("score:%d\tsteps:%d\ttotal:%d\trun:%d" % (score, step, total_step, run))
                    game_model.save_run(score, step, run)
                    break

    def _args(self):
        parser = argparse.ArgumentParser()
        available_games = ["Krakout","Riverraid","Raiders","Renegade","Xecutor","Barbarian"]
        parser.add_argument("-g", "--game", help="Choose from available games: " + str(available_games) + ". Default is 'Riverraid'.", default="Riverraid")
        parser.add_argument("-m", "--mode", help="Choose from available modes: ddqn_train, ddqn_test. Default is 'ddqn_train'.", default="ddqn_train")
        parser.add_argument("-n", "--net", help="Network name to use for testing. Default is 'model'", default="model")
        parser.add_argument("-r", "--render", help="Choose if the game should be rendered. Default is 0 - do not render, 1 - normal, 2 - network vision, 3 - gif file + normal, 4 - featuremap", default=0, type=int)
        parser.add_argument("-tsl", "--total_step_limit", help="Choose how many total steps (frames visible by agent) should be performed. Default is '5000000'.", default=5000000, type=int)
        parser.add_argument("-trl", "--total_run_limit", help="Choose after how many runs we should stop. Default is None (no limit).", default=None, type=int)
        parser.add_argument("-s", "--skip", help="Max random number of frames to skip on reset. Default is 0", default=0, type=int)
        parser.add_argument("-d", "--duel", help="Enable dueling network architecture. Disabled by default",  dest='duel', action='store_true')
        args = parser.parse_args()
        game_mode = args.mode
        model_name = args.net
        game_name = args.game
        render_mode = args.render
        total_step_limit = args.total_step_limit
        total_run_limit = args.total_run_limit
        skip = args.skip
        duel = args.duel
        print ("Selected game: " + str(game_name))
        print ("Selected mode: " + str(game_mode))
        print ("Model name: " + str(model_name))
        print ("Dueling net: " + str(duel))
        print ("Render mode: " + str(render_mode))
        print ("Should skip frames: " + str(skip))
        print ("Total step limit: " + str(total_step_limit))
        print ("Total run limit: " + str(total_run_limit))
        return game_name, game_mode, model_name, render_mode, total_step_limit, total_run_limit, skip, duel

    def _game_model(self, game_mode, model_name, game_name, action_space, duel):
        if game_mode == "ddqn_train":
            return DDQNTrainer(game_name, duel, INPUT_SHAPE, action_space, model_name)
        elif game_mode == "ddqn_test":
            return DDQNSolver(game_name, duel, INPUT_SHAPE, action_space, model_name)
        else:
            print ("Unrecognized mode. Use --help")
            exit(1)

if __name__ == "__main__":
    Main()


text:
from typing import Tuple

import numpy as np
from matplotlib import pyplot as plt

def rotate_point(x: float,
                 y: float,
                 center_x: float,
                 center_y: float,
                 angle_deg: float):
    # Rotate point (x, y) around center point (center_x, center_y)
    # https://en.wikipedia.org/wiki/Rotation_matrix
    angle_radians = angle_deg / 180.0 * np.pi
    sine, cosine = np.sin(angle_radians), np.cos(angle_radians)
    new_x, new_y = np.dot(np.array([x, y]) - np.array([center_x, center_y]),
                          [[cosine, sine],
                           [-sine, cosine]]) + np.array([center_x, center_y])
    return new_x, new_y

def create_2d_figure(figsize: Tuple[float, float],
                     background_color: str,
                     xlim: Tuple[float, float],
                     ylim: Tuple[float, float]):
    fig = plt.figure(figsize=figsize, facecolor=background_color)
    # ax = fig.add_axes([0, 0, 1, 1], frameon=False)
    ax = fig.add_axes([0, 0, 1, 1], frameon=False)
    ax.axis('off')
    ax.set_xlim(xlim)
    ax.set_ylim(ylim)
    return fig, ax

def create_3d_figure(figsize: Tuple[float, float],
                     background_color: str,
                     xlim: Tuple[float, float] = None,
                     ylim: Tuple[float, float] = None,
                     zlim: Tuple[float, float] = None,
                     elev: float = None,
                     azim: float = None,
                     dist: float = None,
                     is_tight_layout: bool = False,
                     proj_type: str = "persp"):
    fig = plt.figure(figsize=figsize, facecolor=background_color,
                     frameon=False)
    # ax = fig.add_subplot(111, projection="3d", proj_type=proj_type)
    ax = fig.add_axes([0, 0, 1, 1], projection="3d", proj_type=proj_type)
    ax.patch.set_facecolor(background_color)
    ax.axis('off')

    if xlim is not None:
        ax.set_xlim(xlim)
    if ylim is not None:
        ax.set_ylim(ylim)
    if zlim is not None:
        ax.set_zlim(zlim)

    if elev is not None:
        ax.elev = elev
    if azim is not None:
        ax.azim = azim
    if dist is not None:
        ax.dist = dist

    if is_tight_layout:
        plt.tight_layout()
    return fig, ax


text:
#!/usr/bin/python
"""
Riley's first research project.
"""
from __future__ import division
import random
from string import ascii_uppercase
import csv
import string_org
import real_value_vector_org
import scipy.stats as stats
import fitness_function as ff
from math import floor
import os
import shutil
import datetime

FITNESS_FUNCTION_TYPE = None
NUMBER_OF_ORGANISMS = None
MUTATION_RATE = None
NUMBER_OF_GENERATIONS = None
OUTPUT_FILE = None
ORG_TYPE = None
TOURNAMENT_SIZE = None
VERBOSE = False
ALTERNATE_ENVIRONMENT_CORR = None
START_TIME = None
CROWDING = False

def create_initial_population():
    population = []
    for _ in range(NUMBER_OF_ORGANISMS):
        if ORG_TYPE == "string":
            population.append(string_org.StringOrg())
        elif ORG_TYPE == "vector":
            population.append(
                real_value_vector_org.RealValueVectorOrg())
    return population

def get_mutated_population(population):
    new_population = []
    for org in population:
        if random.random() < MUTATION_RATE:
            new_org = org.get_mutant()
            new_population.append(new_org)
        else:
            new_population.append(org)
    return new_population

def get_selected_population(population, environment):
    new_population = []
    for _ in range(NUMBER_OF_ORGANISMS):
        orgs = [random.choice(population) for _ in range(TOURNAMENT_SIZE)]
        new_population.append(get_best_organism(orgs, environment))
    return new_population

def get_crowded_population(mutated_population, old_population, environment):
    new_population = []
    for org in mutated_population:
        sample = [random.choice(old_population) for _ in range(TOURNAMENT_SIZE)]
        new_population.append(get_best_crowded_organism(org, sample, environment))
    return new_population

def get_best_organism(pop, environment):
    best_org = pop[0]
    for org in pop:
        if org.is_better_than(best_org, environment):
            best_org = org
    return best_org

def get_best_crowded_organism(new_org, sample, environment):
    most_similar = sample[0]
    min_distance = float("inf")
    for org in sample:
        curr_dist = org.distance(new_org, environment)
        if curr_dist < min_distance:
            most_similar = org
            min_distance = curr_dist
    if new_org.is_better_than(most_similar, environment):
        return new_org
    return most_similar

def get_next_generation(population, environment):
    new_population = get_mutated_population(population)
    if CROWDING:
        new_new_population = get_crowded_population(new_population, population, environment)
    else:
        new_new_population = get_selected_population(new_population, environment)
    return new_new_population

def print_status(generation, population, environment):
    average_fitness = get_average_fitness(population, environment)
    print("Gen = {}  Pop = {}  Fit = {}".format(generation, population, average_fitness))

def evolve_population(reference_environment, alternative_environment):
    current_fitness_list = [("Generation", "Average_Fitness",
                    "Standard_Deviation")]
    current_fitness_best = [("Generation", "Best_fitness", "Best_org")]

    reference_fitness_list = current_fitness_list[:]
    reference_fitness_best = current_fitness_best[:]

    current_environment = reference_environment

    start_of_trimester_2 = int(floor(NUMBER_OF_GENERATIONS/3.0))
    start_of_trimester_3 = int(floor(NUMBER_OF_GENERATIONS*2.0/3.0))

    population = create_initial_population()
    for gen in range(NUMBER_OF_GENERATIONS):
        if gen == start_of_trimester_2:
            current_environment = alternative_environment
        elif gen == start_of_trimester_3:
            current_environment = reference_environment
        population = get_next_generation(population, current_environment)

        average_fitness, stdev, best_org, best_fitness = get_generation_stats(
            population, current_environment)
        current_fitness_list.append((gen, average_fitness, stdev))
        current_fitness_best.append((gen, best_fitness, best_org))

        average_fitness, stdev, best_org, best_fitness = get_generation_stats(
            population, reference_environment)
        reference_fitness_list.append((gen, average_fitness, stdev))
        reference_fitness_best.append((gen, best_fitness, best_org))

        if VERBOSE:
            print_status(gen, population, current_environment)

    result =  (current_fitness_list, current_fitness_best,
               reference_fitness_list, reference_fitness_best)
    return result

def get_generation_stats(population, environment):
    average_fitness = get_average_fitness(population, environment)
    best_org = get_best_organism(population, environment)
    best_fitness = best_org.fitness(environment)
    stdev = stats.tstd([org.fitness(environment)
                        for org in population])
    return average_fitness, stdev, best_org, best_fitness

def get_average_fitness(pop, environment):
    total = 0
    for org in pop:
        total += org.fitness(environment)
    return total / len(pop)

def set_global_variables(config):
    global OUTPUT_FOLDER
    OUTPUT_FOLDER = config.get("DEFAULT", "output_folder")
    global CONFIG_FILE
    CONFIG_FILE = config.get("DEFAULT", "config_file")
    global START_TIME
    START_TIME = config.getfloat("DEFAULT", "start_time")
    global VERBOSE
    VERBOSE = config.getboolean("DEFAULT", "verbose")
    global NUMBER_OF_ORGANISMS
    NUMBER_OF_ORGANISMS = config.getint("DEFAULT", "number_of_organisms")
    global MUTATION_RATE
    MUTATION_RATE = config.getfloat("DEFAULT", "mutation_rate")
    global NUMBER_OF_GENERATIONS
    NUMBER_OF_GENERATIONS = config.getint("DEFAULT", "number_of_generations")
    global TOURNAMENT_SIZE
    TOURNAMENT_SIZE = config.getint("DEFAULT", "tournament_size")
    global ORG_TYPE
    ORG_TYPE = config.get("DEFAULT", "org_type")
    global CROWDING
    CROWDING = eval(config.get("DEFAULT", "crowding"))
    if ORG_TYPE == "string":
        string_org.TARGET_STRING = config.get("DEFAULT", "target_string")
        string_org.LETTERS = config.get("DEFAULT", "letters")
    elif ORG_TYPE == "vector":
        fitness_function_type_str = config.get("DEFAULT", "fitness_function_type")
        global FITNESS_FUNCTION_TYPE
        if fitness_function_type_str == "sphere":
            FITNESS_FUNCTION_TYPE = ff.sphere_function
        elif fitness_function_type_str == "rosenbrock":
            FITNESS_FUNCTION_TYPE = ff.rosenbrock_function
        elif fitness_function_type_str == "rana":
            FITNESS_FUNCTION_TYPE = ff.rana_function
        elif fitness_function_type_str == "deceptive":
            FITNESS_FUNCTION_TYPE = ff.deceptive
        elif fitness_function_type_str == "schafferf7":
            FITNESS_FUNCTION_TYPE = ff.schafferF7
        else:
            raise AssertionError("Unknown (but needed) function type (i.e. sphere)")
        real_value_vector_org.LENGTH = config.getint("DEFAULT", "length")
        range_minimum = config.getint("DEFAULT", "range_minimum")
        real_value_vector_org.RANGE_MIN = range_minimum
        range_maximum = config.getint("DEFAULT", "range_maximum")
        real_value_vector_org.RANGE_MAX = range_maximum
        mut_effect_size = config.getfloat("DEFAULT", "mutation_effect_size")
        ff.MUTATION_EFFECT_SIZE = mut_effect_size
        real_value_vector_org.MUTATION_EFFECT_SIZE = mut_effect_size
    global ALTERNATE_ENVIRONMENT_CORR
    ALTERNATE_ENVIRONMENT_CORR = config.getfloat(
        "DEFAULT", "alternate_environment_corr")

def save_table_to_file(table, filename):
    with open(filename, "wb") as f:
        writer = csv.writer(f)
        writer.writerows(table)

def save_string_to_file(string, filename):
    with open(filename, "w") as f:
        f.write(string)

def generate_data():
    if ORG_TYPE == "vector":
        fitness_function = ff.Fitness_Function(FITNESS_FUNCTION_TYPE, 0, real_value_vector_org.LENGTH)
        fitness_function.create_fitness2(ALTERNATE_ENVIRONMENT_CORR)
        reference_environment  = fitness_function.fitness1_fitness
        alternative_environment  = fitness_function.fitness2_fitness
    elif ORG_TYPE == "string":
        reference_environment  = string_org.default_environment
        alternative_environment  = string_org.hash_environment

    experienced_fits, experienced_bests, reference_fits, reference_bests = evolve_population(
        reference_environment, alternative_environment)

    if os.path.exists(OUTPUT_FOLDER):
        raise IOError("output_folder: {} already exists".format(OUTPUT_FOLDER))
    os.makedirs(OUTPUT_FOLDER)
    config_filename = os.path.basename(CONFIG_FILE)

    def join_path(filename):
        return os.path.join(OUTPUT_FOLDER, filename)

    config_dest = os.path.join(OUTPUT_FOLDER, config_filename)
    shutil.copyfile(CONFIG_FILE, config_dest)

    experienced_filename = join_path("experienced_fitnesses.csv")
    reference_filename = join_path("reference_fitnesses.csv")
    experienced_best_filename = join_path("experienced_best_fitnesses.csv")
    reference_best_filename = join_path("reference_best_fitnesses.csv")
    corr_filename = join_path("correlation.dat")
    time_filename = join_path("time.dat")

    save_table_to_file(experienced_fits, experienced_filename)
    save_table_to_file(reference_fits, reference_filename)
    save_table_to_file(experienced_bests, experienced_best_filename)
    save_table_to_file(reference_bests, reference_best_filename)
    if ORG_TYPE == "vector":
        save_string_to_file(str(fitness_function.correlation()), corr_filename)

    start_time = datetime.datetime.fromtimestamp(START_TIME)
    end_time = datetime.datetime.now()
    time_str = "Start_time {}\nEnd_time {}\nDuration {}\n".format(start_time, end_time, end_time - start_time)
    save_string_to_file(time_str, time_filename)

max_stars_repo_name: emilydolson/ChangingEnvironmentGA

max_issues_repo_name: emilydolson/ChangingEnvironmentGA

max_forks_repo_name: emilydolson/ChangingEnvironmentGA

reason: import scipy


text:
import sys

import h5py
import numpy
from sklearn.model_selection import train_test_split

class BruteForceBLAS():
    def __init__(self, metric, precision=numpy.float32):
        self._metric = metric
        self._precision = precision

    def fit(self, X):
        lens = (X ** 2).sum(-1)
        if self._metric == 'angular':
            X /= numpy.sqrt(lens)[..., numpy.newaxis]
            self.index = numpy.ascontiguousarray(X, dtype=self._precision)
        elif self._metric == 'euclidean':
            self.index = numpy.ascontiguousarray(X, dtype=self._precision)
            self.lengths = numpy.ascontiguousarray(lens, dtype=self._precision)

    def query(self, v, n):
        v = numpy.ascontiguousarray(v, dtype=self._precision)
        if self._metric == 'angular':
            dists = -numpy.dot(self.index, v)
        elif self._metric == 'euclidean':
            dists = self.lengths - 2 * numpy.dot(self.index, v)
        indices = numpy.argpartition(dists, n)[:n]
        return sorted(indices, key=lambda index: dists[index])

def write_output(train, test, fn, distance, point_type='float', count=100):
    f = h5py.File(fn, 'w')
    f.attrs['distance'] = distance
    f.attrs['point_type'] = point_type
    f.create_dataset('train', (len(train), len(train[0])), dtype=train.dtype)[:] = train
    f.create_dataset('test', (len(test), len(test[0])), dtype=test.dtype)[:] = test
    neighbors = f.create_dataset('neighbors', (len(test), count), dtype='i')
    bf = BruteForceBLAS(distance, precision=train.dtype)
    bf.fit(train)
    for i, x in enumerate(test):
        sys.stderr.write('computing queries %d/%d ...\r' % (i+1, len(test)))
        neighbors[i] = bf.query(x, count)

    sys.stderr.write('\n')
    f.close()

def youtube(out_fn):
    fn = './data/youtube.txt'
    X = []
    for line in open(fn):
        v = [float(x) for x in line.strip().split()]
        X.append(numpy.array(v))

    X_train, X_test = train_test_split(X, test_size=10000, random_state=1)
    write_output(numpy.array(X_train), numpy.array(X_test), out_fn, 'angular')

if __name__ == '__main__':
    out_fn = os.path.join('data', 'youtube-40-angular.hdf5')
    youtube(out_fn)


text:
import numpy as np

fname = "day4.txt"
numbers = np.loadtxt(fname, dtype=int, max_rows=1, delimiter=",")
n = np.loadtxt(fname, dtype=int, skiprows=1)
n = n.reshape(-1, n.shape[1], n.shape[1])
board_ids = set(range(n.shape[0]))

def update_board(number, boards, board_ids):
    matches = np.where(boards==number)
    boards[matches] *= -1
    for b,r,c in zip(*matches):
        if b not in board_ids:
            continue
        board = boards[b,:,:]
        if all(board[r,:] <= 0) or all(board[:,c] <= 0):
            board_ids.discard(b)
            if len(board_ids) == 0:
                return np.where(board > 0, board, 0).sum() * number
    return None

for b in numbers:
    res = update_board(b, n, board_ids)
    if res is not None:
        print(b, res)

max_stars_repo_name: boostjanbjorge/adventofcode

max_issues_repo_name: boostjanbjorge/adventofcode

max_forks_repo_name: boostjanbjorge/adventofcode


text:
# Copyright 2018 Recruit Communications Co., Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from pyqubo.array import Array
from pyqubo.integer import Integer
import numpy as np

class LogEncInteger(Integer):
    """Log encoded integer. The value that takes :math:`[0, n]` is
    represented by :math:`\\sum_{i=1}^{\\lceil\\log_{2}n\\rceil}2^ix_{i}` without any constraint.

    Args:
        label (str): Label of the integer.

        lower (int): Lower value of the integer.

        upper (int): Upper value of the integer.

    Examples:
        This example finds the value `a`, `b` such that :math:`a+b=5` and :math:`2a-b=1`.

        >>> from pyqubo import LogEncInteger
        >>> import dimod
        >>> a = LogEncInteger("a", 0, 4)
        >>> b = LogEncInteger("b", 0, 4)
        >>> M=2.0
        >>> H = (2*a-b-1)**2 + M*(a+b-5)**2
        >>> model = H.compile()
        >>> q, offset = model.to_qubo()
        >>> sampleset = dimod.ExactSolver().sample_qubo(q)
        >>> response, broken, e  = model.decode_dimod_response(sampleset, topk=1)[0]
        >>> sol_a = sum(2**k * v for k, v in response["a"].items())
        >>> sol_b = sum(2**k * v for k, v in response["b"].items())
        >>> print("a={},b={}".format(sol_a, sol_b))
        a=2,b=3
    """

    def __init__(self, label, lower, upper):
        assert upper > lower, "upper value should be larger than lower value"
        assert isinstance(lower, int)
        assert isinstance(upper, int)

        self.lower = lower
        self.upper = upper
        self._num_variables = int(np.log2(upper - lower))+1
        self.array = Array.create(label, shape=self._num_variables, vartype='BINARY')
        self.label = label
        self._express = lower + sum(x*2**i for i, x in enumerate(self.array))

    @property
    def express(self):
        return self._express

    @property
    def interval(self):
        return self.lower, self.upper

max_stars_repo_name: OpenJij/pyqubo

max_issues_repo_name: OpenJij/pyqubo

max_forks_repo_name: OpenJij/pyqubo


text:
import tensorflow as tf
import numpy as np
from kerod.layers.anchors import generate_anchors, Anchors
from kerod.core import constants

def get_all_anchors(stride, sizes, ratios, max_size):
    """This function has been taken from tensorpack. https://github.com/tensorpack/tensorpack/blob/master/examples/FasterRCNN/modeling/model_rpn.py#L158

    Get all anchors in the largest possible image, shifted, floatbox

    Arguments:

    - stride (int): the stride of anchors.
    - sizes (tuple[int]): the sizes (sqrt area) of anchors
    - ratios (tuple[int]): the aspect ratios of anchors
    - max_size (int): maximum size of input image

    Returns:

    anchors: SxSxNUM_ANCHORx4, where S == ceil(MAX_SIZE/STRIDE), floatbox
        The layout in the NUM_ANCHOR dim is NUM_RATIO x NUM_SIZE.

    """
    # Generates a NAx4 matrix of anchor boxes in (x1, y1, x2, y2) format. Anchors
    # are centered on 0, have sqrt areas equal to the specified sizes, and aspect ratios as given.
    anchors = []
    for sz in sizes:
        for ratio in ratios:
            w = np.sqrt(sz * sz / ratio)
            h = ratio * w
            anchors.append([-w, -h, w, h])
            cell_anchors = np.asarray(anchors) * 0.5

    field_size = int(np.ceil(max_size / stride))
    shifts = (np.arange(0, field_size) * stride).astype("float32")
    shift_x, shift_y = np.meshgrid(shifts, shifts)
    shift_x = shift_x.flatten()
    shift_y = shift_y.flatten()
    shifts = np.vstack((shift_x, shift_y, shift_x, shift_y)).transpose()

    # Kx4, K = field_size * field_size
    K = shifts.shape[0]

    A = cell_anchors.shape[0]
    field_of_anchors = cell_anchors.reshape((1, A, 4)) + shifts.reshape((1, K, 4)).transpose(
        (1, 0, 2))
    field_of_anchors = field_of_anchors.reshape((field_size, field_size, A, 4))
    # FSxFSxAx4
    # Many rounding happens inside the anchor code anyway
    # assert np.all(field_of_anchors == field_of_anchors.astype('int32'))
    field_of_anchors = field_of_anchors.astype("float32")
    return field_of_anchors

def test_generate_anchors():
    """Ensure the anchor generation is the same than tensorpack for the feature pyramidal network."""
    ratios = [0.5, 1, 2]
    strides = (4, 8, 16, 32, 64)
    scales = (32, 64, 128, 256, 512)
    feature_map_shapes = [200, 100, 50, 25, 12]
    for stride, feature_map_shape in zip(strides, feature_map_shapes):
        anchors_tensorpack = get_all_anchors(stride, scales, ratios, stride * feature_map_shape)

        anchors_od = generate_anchors(stride,
                                      tf.constant(scales, tf.float32),
                                      tf.constant(ratios, tf.float32),
                                      max_size=stride * feature_map_shape)

        anchors_od = tf.gather(anchors_od, [1, 0, 3, 2], axis=-1)
        np.testing.assert_array_almost_equal(anchors_tensorpack, anchors_od, decimal=5)

def test_anchor_layers():
    """Ensure the anchor generation is the same than tensorpack for the feature pyramidal network."""
    ratios = [0.5, 1, 2]
    strides = (4, 8, 16, 32, 64)
    scales = (32, 64, 128, 256, 512)
    feature_maps = [np.zeros((1, shape, shape, 1)) for shape in [200, 100, 50, 25, 12]]

    for stride, feature_map in zip(strides, feature_maps):
        anchors_layer = Anchors(stride, scales, ratios)
        anchors_tensorpack = get_all_anchors(stride, scales, ratios, constants.MAX_IMAGE_DIMENSION)

        anchors_od = anchors_layer(feature_map)
        anchors_od = tf.gather(anchors_od, [1, 0, 3, 2], axis=-1)
        anchors_tensorpack = anchors_tensorpack[:feature_map.shape[1], :feature_map.
                                                shape[2]].reshape(-1, 4)
        np.testing.assert_array_almost_equal(anchors_tensorpack, anchors_od, decimal=5)

max_stars_repo_name: LSanselme/kerod

max_issues_repo_name: LSanselme/kerod

max_forks_repo_name: LSanselme/kerod


text:
import gzip
import inspect
import json
import os
from os.path import join as joinpath
from typing import List, Optional

import numpy as np
from essentia.standard import MetadataReader, Resample
import essentia as es
from joblib import Parallel, delayed
from tqdm import tqdm

from . import utils
from .dataset_utils import chose_score_type, filter
from .idiot import THISDIR

# this only for detecting package directory but breaks readthedocs

# THISDIR = './datasets/'

class Dataset(object):
    def __init__(self,
                 definitions=[joinpath(THISDIR, 'definitions/')],
                 metadataset_path=joinpath(THISDIR, 'datasets.json'),
                 empty=False):
        """
        Load the dataset description and populate the paths

        This object has a fundamental field named `paths` which is a list; each
        entry contain another list of 3 values representing thepath to,
        respectively: mixed recording, signle-sources audio, ground-truth file
        per each source

        Parameters
        ----------

        * definitions : list of str
            paths where `json` dataset definitions are stored; if empty, the
            default definitions are used

        * metadataset_path : str
            the path were the generic information about where this datetimeis
            installed are stored

        * empty : bool
            if True, no definition is loaded

        Returns
        -------
        * AudioScoreDataset :
            instance of the class
        """

        self.datasets = []
        if not empty:
            if len(definitions) == 0:
                definitions = [joinpath(THISDIR, 'definitions/')]
            for path in definitions:
                self.datasets += load_definitions(path)

        # opening medataset json file
        self.metadataset = json.load(open(metadataset_path, 'rt'))
        self.install_dir = self.metadataset['install_dir']
        if self.install_dir.endswith('/'):
            # this shouldn't happen actually...
            self.install_dir = self.install_dir[:-1]

        # self.decompress_path = self.metadataset['decompress_path']
        self.paths = []
        self._chunks = {}

        # let's include all the songs and datasets
        for d in self.datasets:
            d['included'] = True
            for s in d['songs']:
                s['included'] = True
        # populate `paths`
        filter(self)

    def __len__(self):
        return len(self.paths)

    def parallel(self, func, *args, **kwargs):
        """
        Applies a function to all items in `paths` in parallel using
        `joblib.Parallel`.

        You can pass any argument to `joblib.Parallel` by using keyword
        arguments.

        Arguments
        ---------
        func : callable
            the function that will be called; it must accept two arguments
            that are the index of the song and the dataset. Then, it can
            accept all `args` and `kwargs` that are passed to this function:

            >>>  def myfunc(i, dataset, pinco, pal=lino):
            ...     # do not use `filter` and `chunks` here
            ...     print(pinco, pal)
            ...     print(dataset.paths[i])
            ... marco, etto = 4, 5
            ... d = Dataset().filter(datasets='Bach10')
            ... d.parallel(myfunc, marco, n_jobs=8, pal=etto)

            `filter` and `chunks` shouldn't be used.

        Returns
        -------
        list:
            The list of objects returned by each `func`
        """
        joblib_args = [
            k for k, v in inspect.signature(Parallel).parameters.items()
        ]
        joblib_dict = {
            k: kwargs.pop(k)
            for k in dict(kwargs) if k in joblib_args
        }

        return Parallel(**joblib_dict)(delayed(func)(i, self, *args, **kwargs)
                                       for i in tqdm(range(len(self.paths))))

    def get_songs(self):
        """
        Returns a list of dict, each representing a song
        """

        songs = []
        for dataset in self.datasets:
            if dataset['included']:
                for song in dataset['songs']:
                    if song['included']:
                        songs.append(song)
        return songs

    def idx_chunk_to_whole(self, name, idx):
        """
        Given a dataset name and an idx or a list of idx relative to the input
        dataset, returns the idx relative to this whole dataset.

        Use this method if you need, for instance the index of a song for which
        you have the index in a single dataset.
        """
        if type(idx) is int:
            return idx + self._chunks[name][0]
        elif type(idx) is list:
            return [i + self._chunks[name][0] for i in idx]
        else:
            raise Exception('idx should be int or list of int!')

    def get_mix(self, idx, sr=None):
        """
        Returns the audio array of the mixed song

        Arguments
        ---------
        idx : int
            the index of the wanted item
        sr : int or None
            the sampling rate at which the audio will be returned
            (if needed, a resampling is performed). If `None`, no
            resampling is performed

        Returns
        -------
        mix : numpy.ndarray
            the audio waveform of the mixed song
        int :
            The sampling rate of the audio array
        """
        recordings_fn = self.get_mix_paths(idx)

        recordings = []
        for recording_fn in recordings_fn:
            audio, in_sr = utils.open_audio(
                joinpath(self.install_dir, recording_fn))
            recordings.append(audio)

        if len(recordings) > 1:
            L = max(len(rec) for rec in recordings)
            mix = np.zeros(L, dtype=np.float32)
            for rec in recordings:
                mix[:rec.shape[0]] += rec
            mix /= len(rec)
        else:
            mix = recordings[0]

        if sr is not None:
            resampler = Resample(inputSampleRate=in_sr, outputSampleRate=sr)
            mix = resampler(mix)
        else:
            sr = in_sr
        return mix, sr

    def get_gts(self, idx):
        """
        Return the ground-truth of the wanted item

        Arguments
        ---------
        idx : int
            the index of the wanted item

        Returns
        -------
        list :
            list of dictionary representing the ground truth of each single source
        """

        gts = []
        gts_fn = self.get_gts_paths(idx)
        for gt_fn in gts_fn:
            input_fn = joinpath(self.install_dir, gt_fn)

            gt = json.load(gzip.open(input_fn))
            gts.append(gt)
        return gts

    def get_source(self, idx):
        """
        Returns the sources at the specified index

        Arguments
        ---------
        idx : int
            the index of the wanted item

        Returns
        -------
        list :
            a list of numpy.ndarray representing the audio of each source
        int :
            The sampling rate of the audio array
        """
        sources_fn = self.get_sources_paths(idx)

        sources = []
        sr = -1
        for source_fn in sources_fn:
            audio, sr = utils.open_audio(joinpath(self.install_dir, source_fn))
            sources.append(audio)
        return sources, sr

    def get_item(self, idx):
        """
        Returns the mixed audio, sources and ground truths of the specified item.

        Arguments
        ---------
        idx : int
            the index of the wanted item

        Returns
        -------
        numpy.ndarray :
            audio of the mixed sources
        list :
            a list of numpy.ndarray representing the audio of each source
        list :
            list of dictionary representing the ground truth of each single source
        """
        mix = self.get_mix(idx)
        sources = self.get_source(idx)
        gts = self.get_gts(idx)
        return mix, sources, gts

    def get_pianoroll(self,
                      idx,
                      score_type=['misaligned'],
                      resolution=0.25,
                      onsets=False,
                      velocity=True):
        """
        Create pianoroll from list of pitches, onsets and offsets (in this order).

        Arguments
        ---------
        idx : int
            The index of the song to retrieve.
        score_type : list of str
            The key to retrieve the list of notes from the ground_truths. see
            `chose_score_type` for explanation
        resolution : float
            The duration of each column (in seconds)
        onsets : bool
            If True, the value '-1' is put sn each onset
        velocity : bool
            if True, values of each note is the velocity (except the first
            frame if `onsets` is used)

        Returns
        -------
        numpy.ndarray :
            A (128 x n) array where rows represent pitches and columns are time
            instants sampled with resolution provided as argument.

        Note
        ----

        In the midi.org standard, pitches start counting from 0; however,
        sometimes people use to count pitches from 1. Depending on the dataset
        that you are using, verify how pitches are counted. In the ASMD default
        ground-truths, pitches are set with 0-based indexing.

        In case your dataset does not start counting pitches from 0, you should
        correct the output of this function.
        """

        gts = self.get_gts(idx)
        score_type = chose_score_type(score_type, gts)

        # computing the maximum offset
        max_offs = [max(gt[score_type]['offsets']) for gt in gts]
        pianoroll = np.zeros((128, int(max(max_offs) / resolution) + 1))

        # filling pianoroll
        for gt in gts:
            ons = gt[score_type]['onsets']
            offs = gt[score_type]['offsets']
            pitches = gt[score_type]['pitches']
            velocities = gt[score_type]['velocities']
            if not velocities or not velocity:
                velocities = [1] * len(pitches)

            for i in range(len(pitches)):
                p = int(pitches[i])
                on = int(ons[i] / resolution)
                off = int(offs[i] / resolution) + 1

                pianoroll[p, on:off] = velocities[i]
                if onsets:
                    pianoroll[p, on] = -1

        return pianoroll

    def get_beats(self, idx):
        """
        Get a list of beat position in seconds, to be used together with the
        score data.

        Arguments
        ---------
        idx : int
            The index of the song to retrieve.

        Returns
        -------
        numpy.ndarray :
            each row contains beat positions of each ground truth
        """
        gts = self.get_gts(idx)
        beats = []
        for gt in gts:
            beats.append(gt['score']['beats'])

        return np.array(beats)

    def get_score_duration(self, idx):
        """
        Returns the duration of the most aligned score available for a specific
        item
        """
        gts = self.get_gts(idx)
        score_type = chose_score_type(
            ['precise_alignment', 'broad_alignment', 'misaligned', 'score'],
            gts)

        gts_m = 0
        for gt in gts:
            gt_m = max(gt[score_type]['offsets'])
            if gt_m > gts_m:
                gts_m = gt_m
        return gts_m

    def get_audio_data(self, idx):
        """
        Returns audio data of a specific item without loading the full audio.

        N.B. see essentia.standard.MetadataReader!

        Returns
        -------

        list of tuples :
            each tuple is referred to a source and contains the following

        int :
            duration in seconds
        int :
            bitrate (kb/s)
        int :
            sample rate
        int :
            number of channels
        """
        recordings_fn = self.paths[idx][0]

        metadata = []
        for recording_fn in recordings_fn:
            recording_fn = joinpath(self.install_dir, recording_fn)
            reader = MetadataReader(filename=str(recording_fn),
                                    filterMetadata=True)
            metadata.append(reader()[-4:])
        return metadata

    def get_audio(self, idx, sources=None):
        """
        Get the mixed audio of certain sources or of the mix

        Arguments
        ---------
        idx : int
            The index of the song to retrieve.
        sources : list or None
            A list containing the indices of sources to be mixed and returned.
            If `None`, no sources will be mixed and the global mix will be
            returned.

        Returns
        -------
        numpy.ndarray :
            A (n x 1) array which represents the mixed audio.
        int :
            The sampling rate of the audio array
        """

        if sources is not None:
            audio, sr = self.get_source(idx)
            L = max(len(au) for au in audio)
            out = np.zeros(L, dtype=np.float32)
            for i, au in enumerate(audio):
                if i in sources:
                    out[:au.shape[0]] += au
            audio = out / len(audio)
        else:
            audio, sr = self.get_mix(idx)

        return audio, sr

    def get_sources_paths(self, idx) -> List[str]:
        """
        Return paths to single-sources audio recordings, one for each audio

        Returns list of string
        """
        return self.paths[idx][1]

    def get_mix_paths(self, idx) -> List[str]:
        """
        Return paths to the mixed recording if available

        Returns list of string (usually only one)
        """
        return self.paths[idx][0]

    def get_gts_paths(self, idx) -> List[str]:
        """
        Return paths to the ground-truth files, one for each source

        Returns list of string
        """
        return self.paths[idx][2]

    def get_initial_bpm(self, idx) -> Optional[float]:
        """
        Return the initial bpm of the first source if `score` alignment type is
        available at index `idx`, otherwise returns None
        """
        beats = self.get_beats(idx)
        if len(beats[0]) < 2:
            return None

        delta = beats[0][1] - beats[0][0]
        return 60. / delta

    def get_missing_extra_notes(self, idx, kind: str) -> List[np.ndarray]:
        """
        Returns the missing or extra notes of a song. For each source, an array
        of boolean values is returned. If you want the missing/extra notes for
        the whole song, use ``dataset_utils.get_score_mat``

        `kind` can be 'extra' or 'missing'

        """
        gts = self.get_gts(idx)
        out = []
        for gt in gts:
            out.append(np.asarray(gt[kind], dtype=np.bool8))
        return out

def load_definitions(path):
    """
    Given a `path` to a directory, returns a list of dictionaries containing
    the definitions found in that directory (not recursive search)
    """
    datasets = []
    for file in os.listdir(path):
        fullpath = joinpath(path, file)
        if os.path.isfile(fullpath) and fullpath.endswith('.json'):
            # add this dataset
            try:
                datasets.append(json.load(open(fullpath, 'rt')))
            except:
                print("Error opening " + fullpath)
    return datasets

def func_wrapper(func, path, *args, **kwargs):
    d = Dataset(empty=True)
    d.paths = [path]
    return func(0, d, *args, **kwargs)


text:
from __future__ import print_function, division
import matplotlib.pyplot as plt
import tensorflow as tf
import time
import numpy as np
import os
from Code import Utils as u, Nets as n, Losses as l, Penalties as p

class CoGANTrainer(object):

    def __init__(self, g1, g2, d1, d2, domain1, domain2):
        self.hist_g1 = []
        self.hist_g2 = []
        self.hist_d1 = []
        self.hist_d2 = []
        self.hist_semantic_loss = []
        self.hist_cycle_loss = []
        self.hist_weight_similarity = []
        self.hist_discpenalty1 = []
        self.hist_discpenalty2 = []
        self.hist_high_diff = []
        self.hist_low1_diff = []
        self.hist_low2_diff = []
        self.hist_style1_loss = []
        self.hist_style2_loss = []
        self.hist_content_loss = []
        self.X1 = domain1
        self.X2 = domain2
        self.full_training_time = 0
        self.discPenal = p.DiscriminatorPenalties()
        self.genPenal = p.GeneratorPenalties()
        self.content_layers = ['block5_conv2']
        self.style_layers = ['block1_conv1',
                            'block2_conv1',
                            'block3_conv1',
                            'block4_conv1',
                            'block5_conv1']
        self.num_content_layers = len(self.content_layers)
        self.num_style_layers = len(self.style_layers)
        self.vgg_feature_model = None

        self.d1, self.d2 = d1, d2
        self.g1, self.g2 = g1, g2
    def feature_layers(self, layer_names, args):
        vgg = tf.keras.applications.VGG19(include_top=False)
        vgg.trainable = False
        outputs = [vgg.get_layer(name).output for name in layer_names]

        model = tf.keras.Model([vgg.input], outputs)
        return model

    def gram_matrix(self,input):
        result = tf.linalg.einsum('bijc,bijd->bcd', input, input)
        num_locations = input.shape[1] * input.shape[2]
        return result/num_locations

    def StyleContentModel(self, inputs):
        inputs = (0.5 * inputs + 0.5) * 255
        inputs = tf.keras.applications.vgg19.preprocess_input(inputs)
        feature_outputs = self.vgg_feature_model(inputs)
        styles = feature_outputs[:self.num_style_layers]
        content = feature_outputs[self.num_style_layers:]

        style_outputs = [self.gram_matrix(style) for style in styles]

        content_dict = {content_name:value for content_name, value in zip(self.content_layers, content)}
        style_dict = {style_name:value for style_name, value in zip(self.style_layers,style_outputs)}

        return content_dict, style_dict

    def StyleContentLoss(self, style_fake, style_target, content_fake1, content_fake2, args):
        style_loss = tf.add_n([tf.reduce_mean((style_fake[name]-style_target[name])**2) for name in style_fake.keys()])
        content_loss = tf.add_n([tf.reduce_mean((content_fake1[name] - content_fake2[name])**2) for name in content_fake1.keys()])

        style_loss *= args.style_weight / self.num_style_layers
        content_loss *= args.content_weight / self.num_content_layers

        return style_loss, content_loss

    def train(self, args):
        if args.use_cycle:
            self.encoder = n.encoder(args)
        if args.semantic_loss:
            self.classifier = tf.keras.models.load_model(args.classifier_path)
            if args.cogan_data == 'mnist2fashion':
                self.classifier2 = tf.keras.models.load_model(args.classifier_path + '_fashion')
        if args.feature_loss:
            vgg = tf.keras.applications.VGG19(include_top=False, input_shape=(args.dataset_dim[1],args.dataset_dim[2], args.dataset_dim[3]))
            self.high_level_feature_extractor = tf.keras.Model(inputs=vgg.input, outputs=vgg.get_layer('block4_conv4').output)
            self.low_level_feature_extractor = tf.keras.Model(inputs=vgg.input, outputs=vgg.get_layer('block1_pool').output)
        if args.perceptual_loss:
            self.vgg_feature_model = self.feature_layers(self.style_layers + self.content_layers, args)
        it1 = iter(self.X1)
        it2 = iter(self.X2)

        # Set loss functions
        d_loss_fn, g_loss_fn = l.set_losses(args)

        for epoch in range(args.epochs):
            start = time.time()

            # ----------------------
            #  Train Discriminators
            # ----------------------

            for i in range(args.disc_iters):
                # Select a random batch of images
                if args.cogan_data in ['mnist2edge','shapes2flowers', 'Eyeglasses', 'Smiling', 'Blond_Hair', 'Male']:
                    batch1 = next(it1)
                    batch2 = next(it2)
                elif args.cogan_data == 'mnist2svhn_prune':
                    batch1 = next(it1)[0]
                    batch2 = next(it2)
                else:
                    batch1 = next(it1)[0]
                    batch2 = next(it2)[0]

                # Sample noise as generator input
                noise = u.gen_noise(args)

                # Generate a batch of new images
                gen_batch1 = self.g1(noise, training=True)

                # d1
                with tf.GradientTape() as tape:
                    # Disc response
                    disc_real1 = self.d1(batch1, training=True)
                    disc_fake1 = self.d1(gen_batch1[-1], training=True)

                    # Calc loss and penalty
                    d1_loss = d_loss_fn(disc_fake1, disc_real1)
                    gp1 = self.discPenal.calc_penalty(gen_batch1[-1], batch1, self.d1, args)  # if loss is not wgan-gp then gp=0
                    self.hist_discpenalty1.append(gp1)
                    d1_loss = d1_loss + (gp1 * args.penalty_weight_d)
                gradients_of_discriminator = tape.gradient(d1_loss, self.d1.trainable_variables)
                args.disc_optimizer.apply_gradients(zip(gradients_of_discriminator, self.d1.trainable_variables))

                # Generate a batch of new images
                gen_batch2 = self.g2(noise, training=True)

                # d2
                with tf.GradientTape() as tape:
                    # Disc response
                    disc_real2 = self.d2(batch2, training=True)
                    disc_fake2 = self.d2(gen_batch2[-1], training=True)

                    # Calc loss and penalty
                    d2_loss = d_loss_fn(disc_fake2, disc_real2)
                    gp2 = self.discPenal.calc_penalty(gen_batch2[-1], batch2, self.d2, args)  # if loss is not wgan-gp then gp=0
                    self.hist_discpenalty2.append(gp2)
                    d2_loss = d2_loss + (gp2 * args.penalty_weight_d)
                gradients_of_discriminator = tape.gradient(d2_loss, self.d2.trainable_variables)
                args.disc_optimizer.apply_gradients(zip(gradients_of_discriminator, self.d2.trainable_variables))

                if args.loss == 'wgan' and args.disc_penalty == 'none':
                    self.clip_weights(args.clip)

            # ------------------
            #  Train Generators
            # ------------------

            # Sample noise as generator input
            noise = u.gen_noise(args)
            with tf.GradientTape() as tape1, tf.GradientTape() as tape2, tf.GradientTape() as tape3:
                # Adv loss
                gen1_fake = self.g1(noise, training=True)
                disc1_fake = self.d1(gen1_fake[-1], training=True)
                g1_loss = g_loss_fn(disc1_fake)

                gen2_fake = self.g2(noise, training=True)
                disc2_fake = self.d2(gen2_fake[-1], training=True)
                g2_loss = g_loss_fn(disc2_fake)

                if args.semantic_loss:
                    domain1_pred = self.classifier(gen1_fake[-1])
                    domain2_pred = self.classifier(gen2_fake[-1])
                    diff = tf.reduce_mean(tf.math.squared_difference(domain1_pred, domain2_pred))
                    # log semantic loss
                    self.hist_semantic_loss.append(diff)
                    g1_loss = g1_loss + diff * args.semantic_weight
                    g2_loss = g2_loss + diff * args.semantic_weight

                penalty = self.genPenal.calc_penalty(self.g1, self.g2, args.shared_layers, args, gen1_fake, gen2_fake)
                g1_loss = g1_loss + (penalty * args.penalty_weight_g)
                g2_loss = g2_loss + (penalty * args.penalty_weight_g)

                if args.feature_loss:
                    #fake1_high_features = self.high_level_feature_extractor(gen1_fake[-1])
                    #fake2_high_features = self.high_level_feature_extractor(gen2_fake[-1])
                    fake1_low_features = self.low_level_feature_extractor(gen1_fake[-1])
                    fake2_low_features = self.low_level_feature_extractor(gen2_fake[-1])
                    real1_low_features = self.low_level_feature_extractor(batch1)
                    real2_low_features = self.low_level_feature_extractor(batch2)

                    #high_diff = tf.reduce_mean(tf.math.squared_difference(fake1_high_features, fake2_high_features))
                    low_test = tf.reduce_mean(tf.math.squared_difference(fake1_low_features,fake2_low_features))
                    low1_diff = tf.reduce_mean(tf.math.squared_difference(fake1_low_features, real1_low_features))
                    low2_diff = tf.reduce_mean(tf.math.squared_difference(fake2_low_features, real2_low_features))
                    diffs1 = tf.math.l2_normalize([penalty, low1_diff])
                    diffs2 = tf.math.l2_normalize([penalty, low2_diff])

                    #high_diff = diffs1[0] * args.fl_high_weight
                    low1_diff = diffs1[1] * args.fl_low_weight
                    low2_diff = diffs2[1] * args.fl_low_weight

                    #self.hist_high_diff.append(high_diff)
                    self.hist_low1_diff.append(low1_diff)
                    self.hist_low2_diff.append(low2_diff)

                    #g1_loss = g1_loss + high_diff
                    g1_loss = g1_loss + low1_diff
                    #g2_loss = g2_loss + high_diff
                    g2_loss = g2_loss + low2_diff

                    #g1_loss = g1_loss + high_diff + low1_diff
                    #g2_loss = g2_loss + high_diff + low2_diff

                if args.perceptual_loss:
                    fake1_content, fake1_style = self.StyleContentModel(gen1_fake[-1])
                    fake2_content, fake2_style = self.StyleContentModel(gen2_fake[-1])
                    real1_content, real1_style = self.StyleContentModel(batch1)
                    real2_content, real2_style = self.StyleContentModel(batch2)

                    g1_style_loss, g1_content_loss = self.StyleContentLoss(fake1_style, real1_style, fake1_content, fake2_content, args)
                    g2_style_loss, g2_content_loss = self.StyleContentLoss(fake2_style, real2_style, fake2_content, fake1_content, args)

                    g1_loss = (g1_loss) + g1_style_loss + g1_content_loss
                    g2_loss = (g2_loss) + g2_style_loss + g2_content_loss

                    self.hist_style1_loss.append(g1_style_loss)
                    self.hist_style2_loss.append(g2_style_loss)
                    self.hist_content_loss.append(g1_content_loss)

                if args.use_cycle:
                    # Recon loss
                    noise_recon1 = self.encoder(gen1_fake[-1])
                    noise_recon2 = self.encoder(gen2_fake[-1])

                    #fake_recon1 = self.g1(noise_recon1, training=False)
                    #fake_recon2 = self.g2(noise_recon2, training=False)

                    noise_recon_loss1 = l.recon_criterion(noise_recon1, noise)
                    noise_recon_loss2 = l.recon_criterion(noise_recon2, noise)

                    #fake_recon_loss1 = l.recon_criterion(fake_recon1[-1], gen1_fake[-1])
                    #fake_recon_loss2 = l.recon_criterion(fake_recon2[-1], gen2_fake[-1])

                    total_recon_loss = noise_recon_loss1 + noise_recon_loss2

                    # log cycle loss
                    self.hist_cycle_loss.append(total_recon_loss)

                    g1_loss = g1_loss + (total_recon_loss * args.cycle_weight)
                    g2_loss = g2_loss + (total_recon_loss * args.cycle_weight)

            gradients_of_generator1 = tape1.gradient(g1_loss, self.g1.trainable_variables)
            args.gen_optimizer.apply_gradients(zip(gradients_of_generator1, self.g1.trainable_variables))
            gradients_of_generator2 = tape2.gradient(g2_loss, self.g2.trainable_variables)
            args.gen_optimizer.apply_gradients(zip(gradients_of_generator2, self.g2.trainable_variables))
            if args.use_cycle:
                gradients_of_encoder = tape3.gradient(total_recon_loss, self.encoder.trainable_variables)
                args.gen_optimizer.apply_gradients(zip(gradients_of_encoder, self.encoder.trainable_variables))
            weight_sim = self.genPenal.weight_regularizer(self.g1, self.g2, 21)
            self.full_training_time += time.time() - start

            '''
            # Check if shared weights are equal between generators
            a = self.g1.trainable_variables
            b = self.g2.trainable_variables
            mask = []

            for i in range(8):
                if np.array_equal(a[i].numpy(), b[i].numpy()):
                    mask.append(1)
                else:
                    mask.append(0)
            if 0 in mask:
                print("ERROR - weight sharing failure:" + mask)
            '''

            # Collect loss values
            self.hist_d1.append(d1_loss)
            self.hist_d2.append(d2_loss)
            self.hist_g1.append(g1_loss)
            self.hist_g2.append(g2_loss)
            self.hist_weight_similarity.append(weight_sim)

            print("%d [D1 loss: %f] [D2 loss: %f] [G1 loss: %f] [G2 loss: %f] [WeightSim: %f]}" % (epoch, d1_loss, d2_loss, g1_loss, g2_loss, weight_sim))

            # If at save interval => save generated image samples
            if epoch % args.images_while_training == 0:
                self.sample_images(epoch, args.seed, args.dir, args.dataset_dim[3])
        self.plot_losses(args.dir)
        self.sample_images(epoch, args.seed, args.dir, args.dataset_dim[3])
        return self.full_training_time

    def sample_images(self, epoch, seed, dir, channels):
        r, c = 4, 4
        gen_batch1 = self.g1.predict(seed)[-1]
        gen_batch2 = self.g2.predict(seed)[-1]

        gen_imgs = np.concatenate([gen_batch1, gen_batch2])

        # Rescale images 0 - 1
        gen_imgs = 0.5 * gen_imgs + 0.5

        fig, axs = plt.subplots(r, c)
        cnt = 0
        if channels == 1:
            for i in range(r):
                for j in range(c):
                    axs[i, j].imshow(gen_imgs[cnt, :, :, 0], cmap='gray')
                    axs[i, j].axis('off')
                    cnt += 1
            fig.savefig(os.path.join(dir, "images/%d.png" % epoch))
            plt.close()
        # color images
        else:
            for i in range(r):
                for j in range(c):
                    axs[i, j].imshow(gen_imgs[cnt, :, :, :])
                    axs[i, j].axis('off')
                    cnt += 1
            fig.savefig(os.path.join(dir, "images/%d.png" % epoch))
            plt.close()

    def plot_losses(self, dir):
        plt.plot(self.hist_g1, label='Generator 1 loss')
        plt.plot(self.hist_g2, label='Generator 2 loss')
        plt.xlabel('Iterations')
        plt.ylabel('Loss')
        plt.legend()
        plt.savefig(os.path.join(dir, 'losses/gen_loss.png'))
        np.save(os.path.join(dir, 'losses/g1_loss.npy'),self.hist_g1)
        np.save(os.path.join(dir, 'losses/g2_loss.npy'),self.hist_g2)
        plt.close()

        plt.plot(self.hist_d1, label='Discriminator 1 loss')
        plt.plot(self.hist_d2, label='Discriminator 2 loss')
        plt.xlabel('Iterations')
        plt.ylabel('Loss')
        plt.legend()
        plt.savefig(os.path.join(dir, 'losses/disc_loss.png'))
        np.save(os.path.join(dir, 'losses/d1_loss.npy'),self.hist_d1)
        np.save(os.path.join(dir, 'losses/d2_loss.npy'),self.hist_d2)
        plt.close()

        plt.plot(self.hist_discpenalty1, label='DiscPenalty 1')
        plt.plot(self.hist_discpenalty2, label='DiscPenalty 2')
        plt.xlabel('Iterations')
        plt.ylabel('Penalty Value')
        plt.legend()
        plt.savefig(os.path.join(dir, 'losses/disc_penalty.png'))
        np.save(os.path.join(dir, 'losses/d1_penalty.npy'),self.hist_discpenalty1)
        np.save(os.path.join(dir, 'losses/d2_penalty.npy'),self.hist_discpenalty2)
        plt.close()

        plt.plot(self.hist_weight_similarity, label='weight differences')
        plt.xlabel('Iterations')
        plt.ylabel('Difference')
        plt.legend()
        plt.savefig(os.path.join(dir, 'losses/weight_diff.png'))
        np.save(os.path.join(dir, 'losses/weight_diff.npy'),self.hist_weight_similarity)
        plt.close()

        plt.plot(self.hist_semantic_loss, label='semantic loss')
        plt.xlabel('Iterations')
        plt.ylabel('Loss')
        plt.legend()
        plt.savefig(os.path.join(dir, 'losses/semantic_loss.png'))
        np.save(os.path.join(dir, 'losses/semantic_loss.npy'),self.hist_semantic_loss)
        plt.close()

        plt.plot(self.hist_cycle_loss, label='Cycle loss')
        plt.xlabel('Iterations')
        plt.ylabel('Loss')
        plt.legend()
        plt.savefig(os.path.join(dir, 'losses/cycle_loss.png'))
        np.save(os.path.join(dir, 'losses/cycle_loss.npy'),self.hist_cycle_loss)
        plt.close()

        #plt.plot(self.hist_high_diff, label='high_features')
        plt.plot(self.hist_low1_diff, label='low_features_gen1')
        plt.plot(self.hist_low2_diff, label='low_features_gen2')
        plt.xlabel('Iterations')
        plt.ylabel('Difference')
        plt.legend()
        plt.savefig(os.path.join(dir, 'losses/feature_loss.png'))
        plt.close()

        plt.plot(self.hist_style1_loss, label='style1_loss')
        plt.plot(self.hist_style2_loss, label='style2_loss')
        plt.plot(self.hist_content_loss, label='content_loss')
        plt.xlabel('Iterations')
        plt.ylabel('Loss')
        plt.legend()
        plt.savefig(os.path.join(dir, 'losses/perceptual_loss.png'))
        plt.close()

    def clip_weights(self, clip):
        for i, var in enumerate(self.d1.trainable_variables):
            self.d1.trainable_variables[i].assign(tf.clip_by_value(var, -clip, clip))
            #if not np.array_equiv(self.d1.trainable_variables[i].numpy(), self.d2.trainable_variables[i].numpy()):
                #print(i)
        for i, var in enumerate(self.d2.trainable_variables[6:]):
            self.d2.trainable_variables[i + 6].assign(tf.clip_by_value(var, -clip, clip))
            #if not np.array_equiv(self.d1.trainable_variables[i].numpy(), self.d2.trainable_variables[i].numpy()):
                #print(i)


text:
import numpy as np
#Taking Input in Form of list and saving in a list
l=[0]
n=list(map(int,input().split(' ')))
l[0]=n
for i in range(len(n)-1):
    l.append(list(map(int,input().split(' '))))
l=np.array(l)
c=0
for i in range(len(n)):
    if (sum(l[:,i])==(len(n)-1)):
        c=i

if (sum(l[c])==0):
    print("id : "+str(c))
else:
    print("No Celebrity")

max_stars_repo_name: cs-mshah/AlgoCode

max_issues_repo_name: cs-mshah/AlgoCode

max_forks_repo_name: cs-mshah/AlgoCode


text:
#!/usr/bin/env python

import sympy as sp
x, y, z = sp.symbols('x y z')
sp.init_printing(use_unicode=True)

print(sp.diff(x**3 + 5 + 7))

reason: import sympy


text:
import numpy as np
from repeater.results_check.outliers_detection.outliers_detector_decorator import (
    OutliersDetectionDecorator
)

class Quartiles(OutliersDetectionDecorator):
    def __init__(self, outlier_detector, parameters):
        super().__init__(outlier_detector, __name__, parameters)

    def _find_outliers(self, inputs):
        """
        Sources:
        1. Tukey, John W (1977). Exploratory Data Analysis. Addison-Wesley. ISBN 978-0-201-07616-5.
        2. A Review and Comparison of Methods for Detecting Outliers in Univariate Data Sets
        Songwon Seo BS, Kyunghee University, 2002

        The rules of the method are as follows:
        1. The IQR (Inter Quartile Range) is the distance between the lower (Q1) and upper (Q3)
        quartiles.
        2. Inner fences are located at a distance 1.5 IQR below Q1 and above Q3 [Q1-1.5 IQR,
        Q3+1.5 IQR].
        3. Outer fences are located at a distance 3 IQR below Q1 and above Q3 [Q1-3 IQR, Q3+3 IQR].
        4. A value between the inner and outer fences is a possible (minor) outlier.
        An extreme value beyond the outer fences is a probable (major) outlier.

        There is no statistical basis for the reason that Tukey uses 1.5
        and 3 regarding the IQR to make inner and outer fences.

        In this implementation only outer fences are used, because they are probable outlier (step 2 is skipped)
        """
        criterion_used_flag = False
        if self.lower_threshold <= len(inputs) <= self.upper_threshold:
            # find limits of quartiles (step 1)
            first_Q = np.percentile(inputs, 25)
            third_Q = np.percentile(inputs, 75)
            Inter_QD = third_Q - first_Q

            # In this implementation only outer fences are used, because they are probable outlier (step 2 is skipped)

            # find outer limits (step 3)
            Outer_LB = first_Q - Inter_QD * 3
            Outer_HB = third_Q + Inter_QD * 3

            major_outliers = []

            # step 4
            for i in range(0, len(inputs)):
                if inputs[i] < Outer_LB or inputs[i] > Outer_HB:
                    major_outliers.append(inputs[i])

            unique = np.unique(major_outliers)
            criterion_used_flag = True
            return unique, criterion_used_flag
        else:
            return [], criterion_used_flag


text:
import picamera
import time
import cv2
import numpy
import matplotlib.pyplot as plt

class Camera(picamera.PiCamera):
    def __init__(self) -> None:
        pass

print('starting camera...')
cam = picamera.PiCamera()

ipath = '/home/pi/apple/imgn.jpg'

cam.resolution = (256,256) # (64,64) to (2592,1944)
cam.brightness = 70 # 0 to 100
cam.contrast = 70 # 0 to 100
#cam.image_effect = 'sketch'
cam.awb_mode = 'incandescent'
cam.exposure_mode = 'night'
cam.annotate_background = picamera.Color('blue')
cam.annotate_foreground = picamera.Color('yellow')
#cam.annotate_frame_num = 3
#cam.annotate_text = 'test text'
#cam.annotate_text_size = 72
cam.capture(ipath)
#cam.start_recording('/home/pi/apple/vid.h264')
#time.sleep(5)
#cam.stop_recording()
print('camera finished')

print('begin processing')
img = cv2.imread(ipath)
#cv2.imshow('preview',img)
b,g,r = cv2.split(img)
img = cv2.merge((r,g,b))

kernel = numpy.ones((5,5),numpy.uint8)
imgn = cv2.morphologyEx(img, cv2.MORPH_OPEN, kernel)

plt.subplot(121),plt.imshow(img),plt.title('original')
plt.subplot(122),plt.imshow(imgn),plt.title('OPEN')
#plt.set_xlabel('x axis')
#plt.set_ylabel('y axis')
#plt.set_title('title text')
print('finished processing')
plt.show()

max_stars_repo_name: mirrorcoloured/slcypi

max_issues_repo_name: mirrorcoloured/slcypi

max_forks_repo_name: mirrorcoloured/slcypi


text:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from os.path import basename

from . import __constants__ as sc

def _initDF():
    """ Retourne un DataFrame vide avec les colonnes déjà créées """

    columns = [
               'tauR',   # profondeur optique
               'z',      # progondeur géométrique
               'kappaR', # oppacité de Rosseland
               'NH+',    # population H+ (protons)
               'NH',     #            atome hydrogène
               'NHe',    #            hélium
               'NH2',    #            hydrogène moléculaire
               'sm',     # ?????
               'entr',   # entropie
               'P',      # pression
               'T',      # température
               'Ne',     # population électrons
               'rho',    # densité
               'Mass',   # masse au dessus de la couche
               'Frad',   # fraction du flux total en radiation
               'Fconv',  #                        en convection
               'Ftot',   # flux total
              ]

    df = pd.DataFrame(columns = columns)

    return df

def _D2E(s):
    # todo: fix 1.000-100 -> 1.000E-100
    return s.replace(b'D', b'E')

def modele_tlusty6(f):
    """ Lecture des modèles *.6 de TLUSTY
        Retourne un dict avec la température effective, gravité de surface,
        et un DataFrame avec les différents paramètres d'intérêt """

    df = _initDF() # DataFrame vide

    while True:
        # On cherche les lignes d'intérêt
        line = f.readline()
        if 'TEFF'  in line: Teff = float(line.split()[-1])
        if 'LOG G' in line: logg = float(line.split()[-1])
        if 'TOTAL SURFACE FLUX' in line: break # le modèle est en dessous
    line = f.readline() # le header
    f.readline() # ligne vide

    conv = {}
    for i in range(len(line.split())-1):
        # convertisseur 1.D+2 -> 1.E+2, pour chaque colonne
        conv[i] = _D2E

    data = np.loadtxt(f, converters = conv, unpack = True)

    # Stockage dans le DataFrame
    df['Mass']  = data[1]
    df['tauR']  = data[2]
    df['T']     = data[3]
    df['Ne']    = data[4]
    df['rho']   = data[5]
    df['P']     = data[6]
    df['Ftot']  = data[7]
    df['Frad']  = data[8] #* df['Ftot']
    df['Fconv'] = data[9] #* df['Ftot']

    return {'Teff': Teff, 'logg': logg, 'modele': df}

def modele_tlusty7(f):
    """ temp """

    raise NotImplementedError('https://i.imgur.com/10lkUdy.gif')

def modele_test62(f):
    """ Lecture des modèles de test62 / geras
        Retourne un dict avec la température effective, gravité de surface,
        et un DataFrame avec les différents paramètres d'intérêt """

    df = _initDF() # DataFrame vide

    params = f.readline().split() # Paramètres du modèle et atmosphériques

    nq   = int(params[1])
    Teff = float(params[3])
    logg = np.log10(float(params[5]))

    # ligne d'abondances (test62) ou autres paramètres (geras)
    test = f.readline().split()

    if len(test) == 3:
        # modèle geras
        f.readline()

    # pour chaque couche, on lit la ligne
    data = []
    for i in range(nq):
        data.append(np.fromstring(f.readline(), sep = ' '))
    data = np.array(data).T

    # Stockage dans le DataFrame
    df['tauR']   = data[0]
    df['kappaR'] = data[1]
    df['NH+']    = data[2]
    df['NH']     = data[3]
    df['NHe']    = data[4]
    df['NH2']    = data[5]

    f.readline() # Paramètres du modèle et atmosphériques
    f.readline() # Abondances

    # pour chaque couche, on lit la ligne
    data = []
    for i in range(nq):
        data.append(np.fromstring(f.readline(), sep = ' '))
    data = np.array(data).T

    # Stockage dans le DataFrame
    df['sm']     = data[0]
    df['entr']   = data[2]
    df['P']      = data[3]
    df['T']      = data[4]
    df['Ne']     = data[5]

    # calcul de la densité, basé sur l'hydrogène et l'hélium
    density = df[['NH', 'NH+', 'NH2', 'NHe']] * np.array([1, 1, 2, 4])
    df['rho'] = density.sum(axis = 1) / sc.N_A

    # calcul de la profondeur géométrique
    # s = integral( delta_tau / kappa / rho, de 0 à tau )
    kap_rho = df['kappaR'] * df['rho']
    dtau    = df['tauR'].diff()
    df['z'] = (dtau / kap_rho).cumsum()

    return {'Teff': Teff, 'logg': logg, 'modele': df}

def load_modele(inputfile):
    """ Détermine le type de fichier de modèle (TLUSTY, test62 / geras)
        et retourne le dict du modèle """

    with open(inputfile, 'r') as f:

        test = f.readline().split()
        f.seek(0)

        if len(test) == 1:
            # Tlusty.6
            modele = modele_tlusty6(f)

        elif len(test) == 2:
            # Tlusty.7
            modele = modele_tlusty7(f)

        else:
            # test62 / geras
            modele = modele_test62(f)

    return modele

def plot_modele(filelist, figname = None, params = ['T', 'P']):
    """ Trace les structures des modèles dans la liste filelist,
        pour les paramètres dans la liste params.

        Si figname = None, retourne une fenêtre interactive
        Si un string est donné, enregistre la figure sous figname

        params est une liste de deux des différents paramètres du modèle
        d'atmosphère à tracer en fonction de la profondeur optique.
        Par défaut, trace la température et la pression.
    """

    # Labels pour les axes y
    labels  = {
               'tauR'  : r'$\tau_R$',
               'z'     : r'Profondeur géométrique (cm)',
               'kappaR': r'$\kappa_R',
               'NH+'   : r'$N_{H^+}$',
               'NH'    : r'$N_H$',
               'NHe'   : r'$N_{He}$',
               'NH2'   : r'$N_{HII}$',
               'sm'    : r'sm',
               'entr'  : r'S',
               'P'     : r'Pression (dyn cm$^{-2}$)',
               'T'     : r'Température (K)',
               'Ne'    : r'$N_{e^-}$',
               'rho'   : r'Densité (g cm$^{-3}$)',
               'Mass'  : r'Masse (g)',
               'Frad'  : r'Flux radiatif',
               'Fconv' : r'Flux convectif',
               'Ftot'  : r'Flux total',
              }

    if not isinstance(filelist, list):
        filelist = [filelist]

    assert len(params) == 2 # deux paramètres absolument

    # Matplotlib plot
    fig, ax = plt.subplots(nrows = 1, ncols = 2)
    fig.subplots_adjust(wspace=0.3)
    ax[0].minorticks_on()
    ax[1].minorticks_on()

    for i, inputfile in enumerate(filelist):

        # On lit le modèle
        modeledict = load_modele(inputfile)

        Teff   = modeledict['Teff']
        logg   = modeledict['logg']
        modele = modeledict['modele'] # le DataFrame

        for p, param in enumerate(params):

            # Si un des paramètre n'est pas reconnu, on raise une erreur
            if param not in modele.columns:
                raise ValueError('Paramètre {} inconnu.'.format(param))

            if param in ('T', 'z', 'Frad', 'Fconv'):
                # Graphique linéaire en y, log en x
                ax[p].semilogx(modele['tauR'], modele[param],
                             label = basename(inputfile))

            else:
                # Graphique log-log
                ax[p].loglog(modele['tauR'], modele[param],
                             label = basename(inputfile))

            ax[p].set_xlabel(r'$\tau_R$')
            ax[p].set_ylabel(labels[param])

    # Crée la légende
    ax[0].legend()

    if figname is None:
        plt.show(fig)
    else:
        fig.savefig(figname)
    plt.close(fig)

max_stars_repo_name: simonblouin/montreal-py-wd

max_issues_repo_name: simonblouin/montreal-py-wd

max_forks_repo_name: simonblouin/montreal-py-wd