text:
"""
        _                 _
       (_)               | |
 _ __   _  __  __   ___  | |
| '__| | | \ \/ /  / _ \ | |
| |    | |  >  <  |  __/ | |
|_|    |_| /_/\_\  \___| |_|

infinitybatch
-------------

Copyright rixel 2020
Distributed under the MIT License.
See accompanying file LICENSE.
"""

# Modul level constants.
__author__ = 'rixel'
__copyright__ = "Copyright 2020, infinitybatch"
__credits__ = ['rixel']
__license__ = 'MIT'
__version__ = '1.0.0'
__status__ = 'Production'

import torch
from typing import Any, Optional, Union

ERROR_TOLERANCE = 1e-7

class CudaMemorySnapShot(object):
    """
    Provides simple read-only container for memory snapshot of a torch.cuda device.
    -------------------------------------------------------------------------------
    """

    def __init__(self, device_id: Union[torch.device, str, int, None] = None,
                 synchronize: bool = True) -> None:
        """
        Initializes the object
        ----------------------
        @Params: -> device_id: Union[torch.device, str, int, None] = None
                    The device where the snapshot should be taken. If the device
                    is not given it is set to None. The effect of the setting of
                    this parameter fully accords to the behavior of the
                    concerning PyTorch functions.
                 -> synchronize: bool = True
                    Whether or not to apply torch.cuda.synchronize() at the
                    given device before taking the snapshot.
        """

        self.device = device_id
        if synchronize:
            torch.cuda.synchronize(self.device)
        self.total = torch.cuda.get_device_properties(self.device).total_memory
        self.reserved = torch.cuda.memory_reserved(self.device)
        self.allocated = torch.cuda.memory_allocated(self.device)

    @property
    def allocated(self) -> int:
        """
        Property with type of int. It contains the size of the allocated memory
        in bytes at the moment when the snapshot was taken. In fact this
        property is read-only since it can be set at once only, and it is set
        at instantiation. Due to the experimental character of this snapshot,
        trying to give a new value does not raise error however it won't be
        successful.
        -----------
        @Return: -> int
                    The value of allocated.
        """

        return self.__allocated

    @allocated.setter
    def allocated(self, allocated: int) -> None:
        """
        Property with type of int. It contains the size of the allocated memory
        in bytes at the moment when the snapshot was taken. In fact this
        property is read-only since it can be set at once only, and it is set
        at instantiation. Due to the experimental character of this snapshot,
        trying to give a new value does not raise error however it won't be
        successful.
        -----------
        @Params: -> allocated: int
                    The value to set as allocated.
        """

        try:
            self.__allocated
        except AttributeError:
            self.__allocated = allocated

    @property
    def device(self) -> Union[torch.device, str, int, None]:
        """
        Property with type of torch.device, str, int or NoneType. It contains
        the device of the snapshot without converting to torch.device. In fact
        this property is read-only since it can be set at once only, and it is
        set at instantiation. Due to the experimental character of this
        snapshot, trying to give a new value does not raise error however it
        won't be successful.
        --------------------
        @Return: -> Union[torch.device, str, int, None]
                    The value of device.
        """

        return self.__device

    @device.setter
    def device(self, device: Union[torch.device, str, int, None]) -> None:
        """
        Property with type of torch.device, str, int or NoneType. It contains
        the device of the snapshot without converting to torch.device. In fact
        this property is read-only since it can be set at once only, and it is
        set at instantiation. Due to the experimental character of this
        snapshot, trying to give a new value does not raise error however it
        won't be successful.
        --------------------
        @Params: -> device: Union[torch.device, str, int, None]
                    The value to set as device.
        """

        try:
            self.__device
        except AttributeError:
            self.__device = device

    @property
    def free_cache(self) -> int:
        """
        Property with type of int. This is a calculated value of free cache size
        in bytes at the moment when the snapshot was taken. Since this property
        is calculated it is true read-only property.
        --------------------------------------------
        @Return: -> int
        """

        return self.cached - self.allocated

    @property
    def reserved(self) -> int:
        """
        Property with type of int. It contains the size of the reserved memory
        in bytes at the moment when the snapshot was taken. In fact this
        property is read-only since it can be set at once only, and it is set at
        instantiation. Due to the experimental character of this snapshot,
        trying to give a new value does not raise error however it won't be
        successful.
        -----------
        @Return: -> int
                    The value of reserved.
        """

        return self.__reserved

    @reserved.setter
    def reserved(self, reserved: int) -> None:
        """
        Property with type of int. It contains the size of the reserved memory
        in bytes at the moment when the snapshot was taken. In fact this
        property is read-only since it can be set at once only, and it is set at
        instantiation. Due to the experimental character of this snapshot,
        trying to give a new value does not raise error however it won't be
        successful.
        -----------
        @Params: -> reserved: int
                    The value to set as reserved.
        """

        try:
            self.__reserved
        except AttributeError:
            self.__reserved = reserved

    @property
    def total(self) -> int:
        """
        Property with type of int. It contains the size of the total available
        device memory in bytes at the moment when the snapshot was taken. In
        fact this property is read-only since it can be set at once only, and it
        is set at instantiation. Due to the experimental character of this
        snapshot, trying to give a new value does not raise error however it
        won't be successful.
        --------------------
        @Return: -> int
                    The value of total.
        """

        return self.__total

    @total.setter
    def total(self, total: int) -> None:
        """
        Property with type of int. It contains the size of the total available
        device memory in bytes at the moment when the snapshot was taken. In
        fact this property is read-only since it can be set at once only, and it
        is set at instantiation. Due to the experimental character of this
        snapshot, trying to give a new value does not raise error however it
        won't be successful.
        --------------------
        @Params: -> total: int
                    The value to set as total.
        """

        try:
            self.__total
        except AttributeError:
            self.__total = total

class CudaHistory(list):
    """
    Provides a simple container for CudaMemorySnapShot objects with labels.
    -----------------------------------------------------------------------
    Subclass of list.

    Entries of this object are tuples in the form of (label, CudaMemorySnapShot).

    However this is a subclass of list but it cannot be instantiated with
    addition of existing elements like lists can be.
    """

    def __init__(self) -> None:
        """
        Initializes the object
        ----------------------
        """

        super(self.__class__, self).__init__()

    def append(self, name: str,
               device_id: Union[torch.device, str, int, None] = None,
               synchronize: bool = True) -> None:
        """
        Appends items to the container. Every add-in leads to instantiation of
        a new CudaMemorySnapShot object.
        --------------------------------
        @Params: -> name: str
                    The label of the new CudaMemorySnapShot entry.
                 -> device_id: Union[torch.device, str, int, None] = None
                    The device where a new CudaMemorySnapShot should be taken.
                    If the device is not given it is set to None. The effect of
                    the setting of this parameter fully accords to the behavior
                    of the concerning PyTorch functions.
                 -> synchronize: bool = True
                    Whether or not to apply torch.cuda.synchronize() at the
                    given device before taking the snapshot.
        """

        if len(name) > 20:
            name = name[:20]
        super(self.__class__, self).append((name, CudaMemorySnapShot(device_id,
                                                                     synchronize)))

    def __str__(self) -> str:
        """
        Provides human readable representation of the CudaHistory object.
        -----------------------------------------------------------------
        @Return: -> str
                    The representation.
        """

        if len(self) > 0:
            output = 'CudaHistory:\n' + \
                     '| Name                 | Total       | Reserved    | Allocated   |\n' + \
                     '+----------------------+-------------+-------------+-------------+\n'
            for name, snapshot in self:
                output += '| {:20} | {:>11} | {:>11} | {:>11} |\n'.format(name,
                          snapshot.total, snapshot.reserved, snapshot.allocated)
            return output
        else:
            return 'CudaHistory is empty.'

def determinize(seed: Optional[int] = None, be_deterministic: bool = True) -> None:
    """
    Seeds more random sources and cares about the environment to be or don’t be
    deterministic.
    --------------
    @Params: -> seed: Optional[int] = None
                The number to use as seed for manual seeding. Setting this
                number to a constant value highly increases the chance of the
                reproducibility of a training. On the contrary leaving this
                number empty highly increases the randomness of each training
                individually.
             -> be_deterministic: bool = True
                A boolean that specifies whether to switch torch.backends.cudnn
                into deterministic mode or not.

    This function imports random from the standard library and tries to import
    NumPy however it won’t fail if NumPy is not installed.
    """

    import random

    if be_deterministic:
        torch.backends.cudnn.deterministic = True
        torch.backends.cudnn.benchmark = False
        if seed is None:
            seed = 0
    else:
        torch.backends.cudnn.deterministic = False
        torch.backends.cudnn.benchmark = True
    if seed is None:
        if torch.cuda.is_available():
            torch.cuda.seed_all()
        torch.seed()
        try:
            import numpy as np
            np.random.seed()
        except ImportError:
            pass
        random.seed()
    else:
        if torch.cuda.is_available():
            torch.cuda.manual_seed_all(seed)
        torch.manual_seed(seed)
        try:
            import numpy as np
            np.random.seed(seed)
        except ImportError:
            pass
        random.seed(seed)

def is_equal_network(one: torch.nn.Module, another: torch.nn.Module) -> bool:
    """
    Compares two models based on their state_dicts and returns whether their
    weights and biases are exactly equal or not. The bases of the comparison are
    the same layers and same sizes as well.
    ---------------------------------------
    @Params: -> one: torch.nn.Module
                The one model to compare.
             -> another: torch.nn.Module
                The other model to compare.
    @Return: -> bool
                Whether the two networks are the same by weights and biases or not.
    """

    onedict = one.state_dict()
    anotherdict = another.state_dict()
    if len(set(onedict.keys()) ^ set(anotherdict.keys())) == 0:
        for key, data in onedict.items():
            onedata = data.detach()
            anotherdata = anotherdict[key].detach()
            if onedata.device != anotherdata.device:
                onedata = onedata.cpu()
                anotherdata = anotherdata.cpu()
            if not torch.equal(onedata, anotherdata):
                return False
        return True
    else:
        return False

def is_equal_with_error(one: torch.Tensor, another: torch.Tensor,
                        detach: bool=True) -> bool:
    """
    Compares two tensors if they have values in the given calculation error
    tolerance or not. The base of the comparison is the same dimensionality.
    ------------------------------------------------------------------------
    @Params: -> one: torch.Tensor
                One tensor to compare.
             -> another: torch.Tensor
                Another tensor to compare.
                detach: bool=True
                Whether to detach tensors before comparison or not. This
                settings effects on tensors with requires_grad = True only.
    @Return: -> bool
                Whether the two tensors are the same respectively to the
                calculation error tolerance or not.
    """

    if one.shape == another.shape:
        if len(one.shape) != 1:
            if (one.requires_grad or another.requires_grad) and detach:
                manage = lambda atensor: atensor.detach()
            else:
                manage = lambda atensor: atensor
            for i in range(one.shape[0]):
                if not is_equal_with_error(manage(one[i]), manage(another[i]),
                                                                  detach):
                    return False
            return True
        else:
            for i in range(len(one)):
                if compare_with_error_(float(one[i]), float(another[i])):
                    return False
            return True
    else:
        return False

def compare_with_error_(one: float, another: float) -> bool:
    """
    Compares two floats whether they are in the range of the calculation error
    tolerance or not.
    -----------------
    @Params: -> one: float
                One float to compare.
             -> another: float
                Another float to compare.
    @Return: -> bool
                The result of the comparison.
    """

    global ERROR_TOLERANCE

    return abs(one - another) <= ERROR_TOLERANCE

max_stars_repo_name: hyperrixel/infinitybatch

max_issues_repo_name: hyperrixel/infinitybatch

max_forks_repo_name: hyperrixel/infinitybatch


text:
#!/usr/bin/python3
#!encoding=UTF-8

import math
import numpy as np
#import lda

"""
    tools for counting the statistics of things.
    @authors matthew g mcgovern,

"""

def count_symbol_frequency( text ):
    """ Count the frequency of symbol occurances in a body of text.
    """
    frequencies = dict()
    for ch in text:
        if ch in frequencies:
            frequencies[ch] += 1
        else:
            frequencies[ch] = 1
    return frequencies

def count_phrase_frequency( raw_list ):
    """ count the frequency that phrases and words appear in a text,
     when passed the list of phrases (which are kept as lists).
     It's a list of lists.
    """
    frequencies = dict()
    for sub_list in raw_list: #for each sub list in the list
        items = [item for item in sub_list ]
        joined = " ".join( items ) #flatten it into a string
        #and count that ish
        if joined in frequencies:
            frequencies[joined] += 1
        else:
            frequencies[joined] = 1
    return frequencies

def calculate_symbol_entropy( frequency_dict ):
    """
        returns the entropy of a symbol frequency list.
        thiiis might not be correct
    """
    distinct_symbol_count = len( frequency_dict )
    total = sum(frequency_dict.values())
    #print(frequency_dict.values())
    return -sum( [ (frequency/total) * math.log(frequency/total, 2) for frequency in frequency_dict.values() ] )

def generate_list_of_raw_phrases( split_text, up_to_length ):
    """
    Gathers frequency data for n-grams (words and phrases up to length n).
    Pass a body of text and the length N that you want the phrases to be.
    """
    list_of_phrases = []
    for j in range(1, up_to_length+1): #inclusive
        for i in range(0, len(split_text)):
            phrase = split_text[i:i+j]
            if len( phrase) == j:
                list_of_phrases.append( phrase  )
    return list_of_phrases

def count_phrases( text, up_to_length, logographic=False ):
    split_text = None
    """
    if logographic:
        split_text = [ ch for ch in text ]
        #print( split_text )
    """

    split_text = split_with_selector(text, include_dubious=False)
        #print( split_text )
    raw_phrases = generate_list_of_raw_phrases( split_text , up_to_length )
    #print(  raw_phrases )
    phrase_count = count_phrase_frequency( raw_phrases )
    return phrase_count

def collect_ngrams(text, ngram_length=10, single_level=False):
    ngram_dict = dict()

    if single_level:
        for i in range(len(text)):
            substring = None
            try:
                substring = text[i:i+ngram_length]
            except :
                 break;
            #print( substring)
            if substring in ngram_dict:
                ngram_dict[substring] += 1
            else:
                ngram_dict[substring] = 1
    else:
        for length in range(1, ngram_length):
            for i in range(len(text)):
                substring = None
                try:
                    substring = text[i:i+length]
                except :
                     break;
                #print( substring)
                if substring in ngram_dict:
                    ngram_dict[substring] += 1
                else:
                    ngram_dict[substring] = 1
    return ngram_dict

def set_up_character_ranges_table():
    # (low, high, string_name )
    ranges = []
    with open( 'unicode_ranges.txt', 'r') as range_file:
        for line in range_file:
            split = line.split()
            joined = [ int( split[0], 16 ), int(split[1], 16), " ".join(split[2:]) ]
            #print( joined )
            ranges.append( joined )
    return ranges

def check_ranges_from_table( character, table ):
    order = ord(character)
    for i in range( 0 ,len(table)):
        if table[i][0] <= order and order <= table[i][1]:
            #print( character, " is from character set ", table[i][2])
            return table[i][2]

#shoutout to the lda demo code.

"""
def run_lda( matrix ):
    lda_model = lda.LDA(n_topics=50, n_iter=2000, random_state=1)
    lda_model.fit(matrix)
    return lda_model

def print_lda_topic_results( post_run_model, vocabulary, document_titles):
    n_top_words = 9
    for i, topic_distribution in enumerate(model.topic_word_):
        topic_words = np.array(vocabulary)[np.argsort(topic_distribution)][:-n_top_words:-1]
        print('Topics {}: {}'.format(i,' '.join(topic_words)))

def print_lda_document_topic_results( model, vocab, document_titles, how_many ):
    document_topic = model.doc_topic_
    for i in range( how_many ):
        print("{} (top topic: {})".format(titles[i], document_topic[i].argmax()))

"""

from voynich import *

if __name__ == "__main__":
    import sys
    assert( len(sys.argv )> 1  )
    in_file = sys.argv[1]
    with open( in_file, 'r', encoding="latin-1") as inputfile:
        body = ""
        for line in inputfile:
            if line[0] != '#' and parse_line_header(line, transcription='H'):
                line = remove_comments(line)
                line = remove_filler(line)
                line = remove_breaks(line)
                body += remove_line_header(line).strip()
        print( body )
        symb_freq = count_symbol_frequency(body)
        print( 'symbols:', symb_freq )

        entropy = calculate_symbol_entropy(symb_freq)
        #phrases = count_phrases( body, 5, logographic=False )
        #print( phrases )
        ngrams = collect_ngrams(remove_breaks(body, remove_spaces=False), ngram_length=7)
        #print( ngrams )

        ngram_levels = [ collect_ngrams(remove_breaks(body, remove_spaces=False), ngram_length=i, single_level=True) for i in range(1,20) ]
        ngram_entropies = [ calculate_symbol_entropy( level ) for level in ngram_levels ]

        print( "ngram_entropies(1:20)", ngram_entropies )
        print( 'entropy:', entropy )
        print("Found alphabet of %d characters"%len(symb_freq))

"""
    test_text = "Do you love me or do you love me not?" #ima be me
    test_text2 =  "你爱不爱我？"

    print("SYMBOL FREQUENCY:")
    test1 = count_symbol_frequency(test_text)
    print( test1, calculate_symbol_entropy(test1) )

    test2 = count_symbol_frequency(test_text2)
    print( test2, calculate_symbol_entropy(test2 ) )

    print("phrase FREQUENCY:")
    print( count_phrases( test_text2, 5, logographic=True ) )
    print( count_phrases( test_text, 5, logographic=False ) )

    table = set_up_character_ranges_table()
    print("LANGUAGE IDENT FROM UNICODE:")
    print( check_ranges_from_table( '你' , table) )
    print( check_ranges_from_table( 'L' , table) )
    print( check_ranges_from_table( 'ᜊ' , table) )
    print( check_ranges_from_table( 'ʃ' , table) )
    print( check_ranges_from_table( 'к' , table) )
    ##import lda.datasets

    # demo = lda.datasets.load_reuters()
    # vocab = lda.datasets.load_reuters_vocab()
    # titles = lda.datasets.load_reuters_titles()

    # model = run_lda( demo )
    # print_lda_document_topic_results(model, vocab, titles, 10)
    # print_lda_topic_results(model, vocab, titles)

    #print( "woot")
    """

ext: py

max_stars_repo_name: mcgov/yava

max_issues_repo_name: mcgov/yava

max_forks_repo_name: mcgov/yava

reason: import numpy


text:
# coding=utf-8
# Copyright 2022 The Google Research Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Finetune a NMT model with a mixture of in-domain and OOD data.

This script trains a Transformer on a WMT dataset. This runner is
intended for the special case where finetuning data is augmented
with high quality out of domain data.
"""

import functools
import os

from absl import app
from absl import flags
from absl import logging
from clu import metric_writers
from clu import periodic_actions
from flax import jax_utils
from flax import linen as nn
from flax import optim
from flax.training import checkpoints
from flax.training import common_utils
import jax
import jax.numpy as jnp
import numpy as np
import tensorflow as tf

from data_selection.wmt import common
from data_selection.wmt import decode
from data_selection.wmt import input_pipeline
from data_selection.wmt import models
from data_selection.wmt import train_util

FLAGS = flags.FLAGS
flags.adopt_module_key_flags(train_util)

def main(argv):
  if len(argv) > 1:
    raise app.UsageError('Too many command-line arguments.')

  # Make sure tf does not allocate gpu memory.
  tf.config.experimental.set_visible_devices([], 'GPU')

  if FLAGS.jax_backend_target:
    jax.config.FLAGS.jax_xla_backend = 'tpu_driver'
    jax.config.FLAGS.jax_backend_target = FLAGS.jax_backend_target

  # Number of local devices for this host.
  n_devices = jax.local_device_count()

  if jax.process_index() == 0:
    tf.io.gfile.makedirs(FLAGS.model_dir)

  if FLAGS.batch_size % n_devices:
    raise ValueError('Batch size must be divisible by the number of devices')

  vocab_path = FLAGS.vocab_path
  if vocab_path is None:
    vocab_path = os.path.join(FLAGS.model_dir, 'sentencepiece_model')
  tf.io.gfile.makedirs(os.path.split(vocab_path)[0])

  # Load Dataset
  # ---------------------------------------------------------------------------
  logging.info('Initializing dataset.')
  train_ds, eval_ds, predict_ds, encoder = input_pipeline.get_wmt_datasets(
      dataset_name=FLAGS.dataset_name,
      eval_dataset_name=FLAGS.eval_dataset_name,
      shard_idx=jax.process_index(),
      shard_count=jax.process_count(),
      data_dir=FLAGS.data_dir,
      vocab_path=vocab_path,
      target_vocab_size=FLAGS.vocab_size,
      batch_size=FLAGS.batch_size,
      max_length=FLAGS.max_target_length,
      max_eval_length=FLAGS.max_eval_target_length,
      paracrawl_size=FLAGS.paracrawl_size,
      is_scores_path=FLAGS.is_scores_path,
      num_to_keep=FLAGS.data_selection_size,
      pseudo_path=FLAGS.pseudo_path,
      repeat_count=FLAGS.repeat_count,
      newscommentary_size=FLAGS.newscommentary_size,
      split_tokenizer=FLAGS.split_tokenizer)

  if FLAGS.aux_eval_dataset:
    aux_datasets = []
    aux_names = FLAGS.aux_eval_dataset.split(',')
    for name in aux_names:
      _, aux_eval_ds, _, _ = input_pipeline.get_wmt_datasets(
          dataset_name=name,
          eval_dataset_name=None,
          shard_idx=jax.process_index(),
          shard_count=jax.process_count(),
          data_dir=FLAGS.data_dir,
          vocab_path=vocab_path,
          target_vocab_size=FLAGS.vocab_size,
          batch_size=FLAGS.batch_size,
          max_length=FLAGS.max_target_length,
          max_eval_length=FLAGS.max_eval_target_length,
          paracrawl_size=FLAGS.paracrawl_size,
          is_scores_path=FLAGS.is_scores_path,
          num_to_keep=FLAGS.data_selection_size,
          pseudo_path=FLAGS.pseudo_path,
          repeat_count=FLAGS.repeat_count,
          newscommentary_size=FLAGS.newscommentary_size)
      aux_datasets.append(aux_eval_ds)

  train_iter = iter(train_ds)
  vocab_size = int(encoder.vocab_size())
  eos_id = decode.EOS_ID  # Default Sentencepiece EOS token.

  def decode_tokens(toks):
    valid_toks = toks[:np.argmax(toks == eos_id) + 1].astype(np.int32)
    return encoder.detokenize(valid_toks).numpy().decode('utf-8')

  logging.info('Initializing model, optimizer, and step functions.')

  # Build Model and Optimizer
  # ---------------------------------------------------------------------------
  train_config = models.TransformerConfig(
      vocab_size=vocab_size,
      output_vocab_size=vocab_size,
      share_embeddings=FLAGS.share_embeddings,
      logits_via_embedding=FLAGS.logits_via_embedding,
      dtype=jnp.bfloat16 if FLAGS.use_bfloat16 else jnp.float32,
      emb_dim=FLAGS.emb_dim,
      num_heads=FLAGS.num_heads,
      num_layers=FLAGS.num_layers,
      qkv_dim=FLAGS.qkv_dim,
      mlp_dim=FLAGS.mlp_dim,
      max_len=max(FLAGS.max_target_length, FLAGS.max_eval_target_length),
      dropout_rate=FLAGS.dropout_rate,
      attention_dropout_rate=FLAGS.attention_dropout_rate,
      deterministic=False,
      decode=False,
      kernel_init=nn.initializers.xavier_uniform(),
      bias_init=nn.initializers.normal(stddev=1e-6))
  eval_config = train_config.replace(deterministic=True)
  predict_config = train_config.replace(deterministic=True, decode=True)

  start_step = 0
  rng = jax.random.PRNGKey(FLAGS.random_seed)
  rng, init_rng = jax.random.split(rng)
  # It's possible that is supposed to be per device batch size
  input_shape = (FLAGS.batch_size, FLAGS.max_target_length)
  target_shape = (FLAGS.batch_size, FLAGS.max_target_length)

  m = models.Transformer(eval_config)
  initial_variables = jax.jit(m.init)(init_rng,
                                      jnp.ones(input_shape, jnp.float32),
                                      jnp.ones(target_shape, jnp.float32))

  # apply an optimizer to this tree
  optimizer_def = optim.Adam(
      FLAGS.learning_rate,
      beta1=0.9,
      beta2=0.98,
      eps=1e-9,
      weight_decay=FLAGS.weight_decay)
  optimizer = optimizer_def.create(initial_variables['params'])

  # We access model params only from optimizer below via optimizer.target.
  del initial_variables

  if FLAGS.restore_checkpoints:
    logging.info('Restoring checkpoint.')
    # If we have a pretrained model, use that. Else, just continue where leftoff
    model_path = FLAGS.pretrained_model_dir if FLAGS.pretrained_model_dir else FLAGS.model_dir
    optimizer = checkpoints.restore_checkpoint(model_path, optimizer)
    # Grab last step.
    start_step = int(optimizer.state.step)

  writer = metric_writers.create_default_writer(
      FLAGS.model_dir, just_logging=jax.process_index() > 0)

  flag_key = [k for k in FLAGS.flags_by_module_dict().keys() if 'wmt.par' in k
             ]
  if flag_key:
    flag_key = flag_key[0]
    local_flags = {
        f.name: f.value for f in FLAGS.flags_by_module_dict()[flag_key]
    }
    writer.write_hparams(local_flags)

  # Replicate optimizer.
  optimizer = jax_utils.replicate(optimizer)

  learning_rate_fn = common.create_learning_rate_scheduler(
      base_learning_rate=FLAGS.learning_rate, warmup_steps=FLAGS.warmup_steps,
      steps_per_cycle=FLAGS.steps_per_cycle, init_step=start_step,
      finetune_lr=FLAGS.finetune_lr)

  # compile multidevice versions of train/eval/predict step and cache init fn.
  p_train_step = jax.pmap(
      functools.partial(
          train_util.train_step,
          config=train_config,
          learning_rate_fn=learning_rate_fn,
          label_smoothing=FLAGS.label_smoothing),
      axis_name='batch',
      donate_argnums=(0,))  # pytype: disable=wrong-arg-types
  p_eval_step = jax.pmap(
      functools.partial(train_util.eval_step, config=eval_config),
      axis_name='batch')
  p_init_cache = jax.pmap(
      functools.partial(
          train_util.initialize_cache,
          max_decode_len=FLAGS.max_predict_length,
          config=predict_config),
      axis_name='batch')
  p_pred_step = jax.pmap(
      functools.partial(
          train_util.predict_step,
          config=predict_config,
          beam_size=FLAGS.beam_size),
      axis_name='batch',
      static_broadcasted_argnums=(3, 4))  # eos token, max_length are constant

  # Main Train Loop
  # ---------------------------------------------------------------------------

  # We init the first set of dropout PRNG keys, but update it afterwards inside
  # the main pmap"d training update for performance.
  dropout_rngs = jax.random.split(rng, jax.local_device_count())
  del rng

  logging.info('Starting training loop.')
  hooks = []
  report_progress = periodic_actions.ReportProgress(
      num_train_steps=FLAGS.num_train_steps, writer=writer)
  if jax.process_index() == 0:
    hooks += [
        report_progress,
        periodic_actions.Profile(logdir=FLAGS.model_dir, num_profile_steps=5)
    ]
  train_metrics = []
  total_steps = start_step + FLAGS.num_train_steps
  if FLAGS.eval_only:
    total_steps = start_step + 1
  best_eval_loss = 1000
  curr_eval_loss = 1000
  eval_loss_history = []
  do_resample_data = False
  gradual_selection_size = FLAGS.data_selection_size
  eval_freq = FLAGS.eval_frequency
  with metric_writers.ensure_flushes(writer):
    for step in range(start_step, total_steps):
      is_last_step = step == total_steps - 1

      # Resample training data for gradual FT
      if do_resample_data:
        # resample data
        do_resample_data = False
        if eval_loss_history[-1] > eval_loss_history[-2]:
          gradual_selection_size = int(gradual_selection_size / .75)
        else:
          gradual_selection_size = int(.75 * gradual_selection_size)
        if gradual_selection_size < 500_000:
          eval_freq = int(gradual_selection_size) / 100

        train_ds, eval_ds, predict_ds, encoder = input_pipeline.get_wmt_datasets(
            dataset_name=FLAGS.dataset_name,
            eval_dataset_name=FLAGS.eval_dataset_name,
            shard_idx=jax.process_index(),
            shard_count=jax.process_count(),
            data_dir=FLAGS.data_dir,
            vocab_path=vocab_path,
            target_vocab_size=FLAGS.vocab_size,
            batch_size=FLAGS.batch_size,
            max_length=FLAGS.max_target_length,
            max_eval_length=FLAGS.max_eval_target_length,
            paracrawl_size=FLAGS.paracrawl_size,
            is_scores_path=FLAGS.is_scores_path,
            num_to_keep=gradual_selection_size,
            pseudo_path=FLAGS.pseudo_path,
            repeat_count=FLAGS.repeat_count,
            newscommentary_size=FLAGS.newscommentary_size,
            split_tokenizer=FLAGS.split_tokenizer)
        train_iter = iter(train_ds)
        logging.info('Decrease selection size to %d at step %d',
                     gradual_selection_size, step)

      # Shard data to devices and do a training step.
      if not FLAGS.eval_only:
        with jax.profiler.StepTraceAnnotation('train', step_num=step):
          try:
            batch = common_utils.shard(
                jax.tree_map(np.asarray, next(train_iter)))
            optimizer, metrics = p_train_step(
                optimizer, batch, dropout_rng=dropout_rngs)
            train_metrics.append(metrics)
          except StopIteration:
            is_last_step = True

      # Quick indication that training is happening.
      logging.log_first_n(logging.INFO, 'Finished training step %d.', 5, step)
      for h in hooks:
        h(step)

      # Periodic metric handling.
      if (step - start_step) % eval_freq == 0 or is_last_step:
        if not FLAGS.eval_only:
          with report_progress.timed('training_metrics'):
            logging.info('Gathering training metrics.')
            train_metrics = common_utils.get_metrics(train_metrics)
            lr = train_metrics.pop('learning_rate').mean()
            metrics_sums = jax.tree_map(jnp.sum, train_metrics)
            denominator = metrics_sums.pop('denominator')
            summary = jax.tree_map(lambda x: x / denominator, metrics_sums)  # pylint: disable=cell-var-from-loop
            summary['learning_rate'] = lr
            summary = {'train_' + k: v for k, v in summary.items()}
            writer.write_scalars(step, summary)
            train_metrics = []

        if FLAGS.eval_only:
          p_eval_per_pos_step = jax.pmap(
              functools.partial(
                  train_util.eval_per_pos_step, config=eval_config),
              axis_name='batch')
          # Get per example loss
          loss_filename = FLAGS.model_dir + '/test_losses.csv'
          train_util.write_per_example_losses(
              p_eval_step=p_eval_per_pos_step,
              target=optimizer.target,
              eval_ds=eval_ds,
              num_eval_steps=FLAGS.num_eval_steps,
              loss_filename=loss_filename)
        else:
          with report_progress.timed('eval'):
            eval_results = train_util.evaluate(
                p_eval_step=p_eval_step,
                target=optimizer.target,
                eval_ds=eval_ds,
                num_eval_steps=FLAGS.num_eval_steps)
            curr_eval_loss = eval_results['loss']
            eval_loss_history.append(curr_eval_loss)
            if len(eval_loss_history) > 1:
              orig_loss = eval_loss_history[-2]
              percent_change = (orig_loss - curr_eval_loss) / orig_loss
              percent_change *= 100
              if percent_change < .1:
                do_resample_data = True
            writer.write_scalars(
                step, {'eval_' + k: v for k, v in eval_results.items()})

        if FLAGS.aux_eval_dataset:
          for aux_i, aux_eval_ds in enumerate(aux_datasets):
            with report_progress.timed('aux_eval'):
              eval_results = train_util.evaluate(
                  p_eval_step=p_eval_step,
                  target=optimizer.target,
                  eval_ds=aux_eval_ds,
                  num_eval_steps=FLAGS.num_eval_steps)
              writer.write_scalars(
                  step, {
                      'aux' + str(aux_i) + '_eval_' + k: v
                      for k, v in eval_results.items()
                  })

        if FLAGS.compute_bleu:
          with report_progress.timed('translate_and_bleu'):
            decode_file = FLAGS.model_dir + '/decodes.csv'
            exemplars, bleu_score = train_util.translate_and_calculate_bleu(
                p_pred_step=p_pred_step,
                p_init_cache=p_init_cache,
                target=optimizer.target,
                predict_ds=predict_ds,
                decode_tokens=decode_tokens,
                max_predict_length=FLAGS.max_predict_length,
                num_eval_steps=FLAGS.num_eval_steps,
                decode_file=decode_file if FLAGS.eval_only else '')
            writer.write_scalars(step, {'bleu': bleu_score})
            writer.write_texts(step, {'samples': exemplars})

      # Save a checkpoint on one host after every checkpoint_freq steps.
      save_checkpoint = ((step - start_step) % FLAGS.checkpoint_freq == 0 or
                         is_last_step)
      if FLAGS.save_checkpoints and save_checkpoint and jax.process_index(
      ) == 0:
        if curr_eval_loss < best_eval_loss:  # only save better checkpoints
          best_eval_loss = curr_eval_loss
          with report_progress.timed('checkpoint'):
            checkpoints.save_checkpoint(
                FLAGS.model_dir, jax_utils.unreplicate(optimizer),
                step, keep=FLAGS.chkpts_to_keep, overwrite=True)

      if is_last_step:
        break

if __name__ == '__main__':
  app.run(main)

reason: import numpy,import jax


text:
import numpy as np
import time

import utils

class RandomSearchRlAlg():
    def __init__(self, deltas, num_deltas, deltas_used):
        self.deltas = deltas
        self.num_deltas = num_deltas  # N
        self.deltas_used = deltas_used  # b

"""
Ok I think this interface makes sense.
The only thing to change would be to use the replay buffer in the agent.
"""
class ARS(RandomSearchRlAlg):
    def __init__(self, deltas, num_deltas, deltas_used):
        super(ARS, self).__init__(deltas, num_deltas, deltas_used)

    def compute_error(self, deltas_idx, rollout_rewards, w_policy):
        # select top performing directions if deltas_used < num_deltas
        max_rewards = np.max(rollout_rewards, axis = 1)

        if self.deltas_used > self.num_deltas:
            self.deltas_used = self.num_deltas

        idx = np.arange(max_rewards.size)[max_rewards >= np.percentile(max_rewards, 100*(1 - (self.deltas_used / self.num_deltas)))]
        deltas_idx = deltas_idx[idx]
        rollout_rewards = rollout_rewards[idx,:]

        # normalize rewards by their standard deviation
        if np.std(rollout_rewards) > 0:
            rollout_rewards /= np.std(rollout_rewards)

        t1 = time.time()
        # aggregate rollouts to form g_hat, the gradient used to compute SGD step
        g_hat, count = utils.batched_weighted_sum(rollout_rewards[:,0] - rollout_rewards[:,1],
                                                  (self.deltas.get(idx, w_policy.size)
                                                   for idx in deltas_idx),
                                                  batch_size = 500)
        g_hat /= deltas_idx.size
        return g_hat

    def improve(self, agent, deltas_idx, rollout_rewards):
        t1 = time.time()
        g_hat = self.compute_error(deltas_idx, rollout_rewards, agent.weights)
        print("\tEuclidean norm of update step:", np.linalg.norm(g_hat))
        agent.weights -= agent.optimizer._compute_step(g_hat).reshape(agent.weights.shape)
        t2 = time.time()
        print('\t\tTime to update weights', t2-t1)

max_stars_repo_name: mbchang/ARS

max_issues_repo_name: mbchang/ARS

max_forks_repo_name: mbchang/ARS


text:
import tensorflow as tf
import keras_preprocessing
from keras_preprocessing import image
from keras_preprocessing.image import ImageDataGenerator
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import os
import numpy as np

rock_dir = os.path.join('/Users/seanjudelyons/Downloads/Rock Paper Scissors/rps/rock/')
paper_dir = os.path.join('/Users/seanjudelyons/Downloads/Rock Paper Scissors/rps/paper/')
scissors_dir = os.path.join('/Users/seanjudelyons/Downloads/Rock Paper Scissors/rps/scissors/')

print('total training rock images:', len(os.listdir(rock_dir)))
print('total training paper images:', len(os.listdir(paper_dir)))
print('total training scissors images:', len(os.listdir(scissors_dir)))

rock_files = os.listdir(rock_dir)
print(rock_files[:10])

paper_files = os.listdir(paper_dir)
print(paper_files[:10])

scissors_files = os.listdir(scissors_dir)
print(scissors_files[:10])

pic_index = 2

next_rock = [os.path.join(rock_dir, fname)
                for fname in rock_files[pic_index-2:pic_index]]
next_paper = [os.path.join(paper_dir, fname)
                for fname in paper_files[pic_index-2:pic_index]]
next_scissors = [os.path.join(scissors_dir, fname)
                for fname in scissors_files[pic_index-2:pic_index]]

for i, img_path in enumerate(next_rock+next_paper+next_scissors):
  #print(img_path)
  img = mpimg.imread(img_path)
  plt.imshow(img)
  plt.axis('Off')
  plt.show()

TRAINING_DIR = "/Users/seanjudelyons/Downloads/Rock Paper Scissors/rps/"
training_datagen = ImageDataGenerator(
      rescale = 1./255,
	    rotation_range=40,
      width_shift_range=0.2,
      height_shift_range=0.2,
      shear_range=0.2,
      zoom_range=0.2,
      horizontal_flip=True,
      fill_mode='nearest')

VALIDATION_DIR = "/Users/seanjudelyons/Downloads/Rock Paper Scissors/rps-validation/"
validation_datagen = ImageDataGenerator(rescale = 1./255)

train_generator = training_datagen.flow_from_directory(
	TRAINING_DIR,
	target_size=(150,150),
	class_mode='categorical',
  batch_size=126
)

validation_generator = validation_datagen.flow_from_directory(
	VALIDATION_DIR,
	target_size=(150,150),
	class_mode='categorical',
  batch_size=126
)

model = tf.keras.models.Sequential([
    # Note the input shape is the desired size of the image 150x150 with 3 bytes color
    # This is the first convolution
    tf.keras.layers.Conv2D(64, (3,3), activation='relu', input_shape=(150, 150, 3)),
    tf.keras.layers.MaxPooling2D(2, 2),
    # The second convolution
    tf.keras.layers.Conv2D(64, (3,3), activation='relu'),
    tf.keras.layers.MaxPooling2D(2,2),
    # The third convolution
    tf.keras.layers.Conv2D(128, (3,3), activation='relu'),
    tf.keras.layers.MaxPooling2D(2,2),
    # The fourth convolution
    tf.keras.layers.Conv2D(128, (3,3), activation='relu'),
    tf.keras.layers.MaxPooling2D(2,2),
    # Flatten the results to feed into a DNN
    tf.keras.layers.Flatten(),
    tf.keras.layers.Dropout(0.5),
    # 512 neuron hidden layer
    tf.keras.layers.Dense(512, activation='relu'),
    tf.keras.layers.Dense(3, activation='softmax')
])

model.summary()

model.compile(loss = 'categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])

history = model.fit(train_generator, epochs=10, steps_per_epoch=20, validation_data = validation_generator, verbose = 1, validation_steps=3)

model.save("mymodel.h5")

path = os.path.join('/Users/seanjudelyons/Downloads/Rock Paper Scissors/rps-test-set/rock/testrock01-26.png')
img = image.load_img(path, target_size=(150, 150))
x = image.img_to_array(img)
x = np.expand_dims(x, axis=0)

images = np.vstack([x])
classes = model.predict(images, batch_size=10)
print(classes)

max_stars_repo_path: Rock Paper and Scissor.py

max_issues_repo_path: Rock Paper and Scissor.py

max_forks_repo_path: Rock Paper and Scissor.py


text:
#%% PART 1
import numpy as np
import matplotlib.pyplot as plt
import timeit
import math

y_0 = 1.5
theta = np.radians(13.79483459)
g = 9.81
v_0 = 30.72081747
x = np.linspace(0,70,1000)

def y(x):
    return (np.tan(theta))*x - (g)/(2*v_0**2*(np.cos(theta))**2)*x**2 + y_0

plt.plot(x, y(x))
plt.axhline(0)

def newton(f,df,x,epsilon,max_iter):
    for n in range(0,max_iter):
        if abs(f(x)) < epsilon:
            print('Found solution after',n,'iterations.')
            return x
        if Df(x) == 0:
            print('Zero derivative. No solution found.')
            return None
        x = x - f(x)/Df(x)
    print('Exceeded maximum iterations. No solution found.')
    return None

f = lambda x: (np.tan(theta))*x - (g)/(2*v_0**2*(np.cos(theta))**2)*x**2 + y_0
Df= lambda x: (np.tan(theta)) - 2*(g)/(2*v_0**2*(np.cos(theta))**2)*x
x = 50
epsilon=0.01
max_iter=1000
start = timeit.default_timer()
solution = newton(f,Df,x,epsilon,max_iter)
stop = timeit.default_timer()
print('Time of changed: ', stop - start)
print(f'The position at which the projectile hits the ground is at x: {solution}')

#%% PART 2
# Using Newton Raphson again to find the maximum height reached

def fprime(x):
    return (np.tan(theta)) - 2*(g)/(2*v_0**2*(np.cos(theta))**2)*x
def fsecond(x):
    return  - 2*(g)/(2*v_0**2*(np.cos(theta))**2)
def quadratic_approx(x, x0, f, fprime, fsecond):
    return f(x0)+fprime(x0)*(x-x0)+0.5*fsecond(x0)*(x-x0)**2
def newton2(x0, fprime, fsecond, maxiter=100, eps=0.0001):
    x=x0
    for i in range(maxiter):
        xnew=x-(fprime(x)/fsecond(x))
        if xnew-x<eps:
            return xnew
            print('converged')
            break
        x = xnew
    return x

x_star=newton2(0, fprime, fsecond)
print(f'The maximum height is reached at x: {x_star}')

#%%PART 3
# Finding the sensitivity of the landing spot

theta = np.radians(31)
solution2 = newton(f,Df,x,epsilon,max_iter)
print(f'The position at which the projectile hits the ground is at x: {solution2} and theta: {theta}')
theta2 = theta

theta = np.radians(30)
solution3 = newton(f,Df,x,epsilon,max_iter)
print(f'The position at which the projectile hits the ground is at x: {solution3} and theta: {theta}')
theta3 = theta

theta = np.radians(29)
solution4 = newton(f,Df,x,epsilon,max_iter)
theta4 = theta
print(f'The position at which the projectile hits the ground is at x : {solution4} and theta: {theta}')
print(f'An average change of theta: {(theta2-theta3+theta3-theta4)/2} equals to a change in height of: {(solution3-solution4 + solution2-solution3)/2}')

#%%PART 4
# Optimum angle and initial pitch velocity to get the projectile to hit the ground to 50 m

# DOING IT IN EXCEL


text:
import numpy as np
import torch
from torchvision import transforms
from torchvision.transforms import functional as tf
import pickle

from config import MemoryConfigPIL, global_config
from replay_memory.replay_memory import Memory

class ReplayPILDataset(Memory):
    """
    A DIY memory class where images are saved in PIL format, allowing for flipping and color jittering
    as data augmentation during training. Actual memory use is not very efficient/fast.
    """

    def __init__(self, memory_config: MemoryConfigPIL):
        super(ReplayPILDataset, self).__init__()
        self.prioritized_replay = memory_config.prioritized_replay
        self.config = memory_config
        self.images = [None] * memory_config.capacity
        self.depths = [None] * memory_config.capacity if global_config.depth else None
        self.targets = [None] * memory_config.capacity
        self.superpixels = [None] * memory_config.capacity if global_config.superpixels else None
        self.priorities = np.zeros(memory_config.capacity) if self.prioritized_replay else None
        self.last = 0
        self.max = 0
        self.to_pil = transforms.ToPILImage()
        self.to_tensor = transforms.ToTensor()
        self.image_transform = transforms.Compose([
            transforms.RandomApply([transforms.ColorJitter(*(memory_config.jitter,) * 4)],
                                   p=memory_config.jitter_prob), self.to_tensor])
        self.depth_image_transform = transforms.ToTensor()
        self.batch_size = None

    def add_batch(self, batch):
        if global_config.superpixels:
            images, targets, superpixels = batch
        else:
            images, targets = batch
            superpixels = [None] * len(images)

        for image, target, superpixel in zip(images, targets, superpixels):
            if self.last == self.config.capacity:
                self.last = 0

            if global_config.depth:
                image, depth = image
                self.images[self.last] = self.to_pil(image)
                self.depths[self.last] = self.to_pil(depth)
            else:
                self.images[self.last] = self.to_pil(image)

            self.targets[self.last] = target

            if global_config.superpixels:
                self.superpixels[self.last] = superpixel

            if self.prioritized_replay:
                self.priorities[self.last] = self.config.initial_priority

            self.last += 1
            self.max = max(self.max, self.last)

    def initialize_loader(self, batch_size):
        self.batch_size = batch_size

    def iterator(self):
        probs = self.priorities[:self.max] / self.priorities[:self.max].sum() if self.prioritized_replay else None
        index_batches = np.random.choice(self.max, self.max // self.batch_size * self.batch_size,
                                         replace=False, p=probs).reshape((-1, self.batch_size))
        return (self.make_batch(indices) for indices in index_batches)

    def make_batch(self, indices):
        images, targets = [], []
        superpixels = [] if global_config.superpixels else None
        for i in indices:
            image = (self.images[i], self.depths[i]) if global_config.depth else self.images[i]

            image, target, superpixel = self.flip(image, self.targets[i],
                                                  self.superpixels[i] if global_config.superpixels else None)

            if global_config.depth:
                image = torch.cat([self.image_transform(image[0]).float(),
                                   self.depth_image_transform(image[1])], dim=0)
            else:
                image = self.image_transform(image).float()

            images.append(image)

            target = [torch.from_numpy(t) for t in target]
            targets.append(target)

            if global_config.superpixels:
                superpixels.append(torch.from_numpy(superpixel))

        images = torch.stack(images)
        targets = (list(x) for x in zip(*targets))
        targets = tuple(torch.stack(x) for x in targets)

        batch = dict(images=images, targets=targets)

        if global_config.superpixels:
            batch['superpixels'] = torch.stack(superpixels)

        if self.prioritized_replay:
            batch['indices'] = indices
        if self.prioritized_replay and self.config.bias_correct:
            batch['weights'] = torch.tensor([1/self.priorities[i] for i in indices])
        else:
            batch['weights'] = torch.ones(len(indices))

        return batch

    def flip(self, image, target, superpixel):
        if np.random.random(1) < self.config.flip_prob:
            if global_config.depth:
                image = (tf.hflip(image[0]), tf.hflip(image[1]))
            else:
                image = tf.hflip(image)

            target = tuple(np.flip(t, axis=-1).copy() for t in target)

            if global_config.superpixels:
                superpixel = np.flip(superpixel, axis=-1).copy()

        return image, target, superpixel

    def load_memory(self, path):
        arrays = pickle.load(open(path, 'rb'))

        images, targets = arrays[0][-self.config.capacity:], arrays[1][-self.config.capacity:]
        size = len(images)
        self.images[:size] = images
        self.targets[:size] = targets

        if global_config.depth:
            depths = arrays[2][-self.config.capacity:]
            self.depths[:size] = depths
        if global_config.superpixels:
            superpixels = arrays[3][-self.config.capacity:]
            self.superpixels[:size] = superpixels

        self.max = size
        self.last = size

    def save_memory(self, path):
        arrays = (self.images[:self.max], self.targets[:self.max])
        if global_config.depth:
            arrays = arrays + (self.depths[:self.max],)
        if global_config.superpixels:
            arrays = arrays + (self.superpixels[:self.max],)
        file = open(path, 'wb')
        pickle.dump(arrays, file)
        file.close()

    def base_image_transform(self, image):
        if global_config.depth:
            image = tuple(self.to_tensor(self.to_pil(im)) for im in image)
        else:
            image = self.to_tensor(self.to_pil(image))

        if global_config.depth:
            image = torch.cat(list(image), dim=0)

        return image

    def update_priorities(self, batch_indices: list, priorities: list):
        for index, priority in zip(batch_indices, priorities):
            self.priorities[index] = priority


text:
import numpy as np
from PIL import Image
from os import listdir
from os.path import isfile
from tensorflow.keras.utils import to_categorical

def load_images(train_dir_path, val_dir_path):

    ###############################################
    ### Loads the Images in the PVTL Subfolders ###
    ###############################################

    dir_paths = [train_dir_path, val_dir_path]

    labels_dict = {'vehicle-red': 0, 'vehicle-green': 1, 'pedestrian-red': 2, 'pedestrian-green': 3, 'others': 4}

    lbs = list(labels_dict.keys())

    X_train = []
    y_train = []
    X_val = []
    y_val = []

    for e in range(len(dir_paths)):
        for l in lbs:
            if e == 0:
                p = dir_paths[e] + "/" + l + "/"
                image_paths = [p + f for f in listdir(p) if isfile(p + f)]

                for img_p in image_paths:
                    temp = np.asarray(Image.open(img_p))
                    X_train.append(temp)
                y_train = y_train + [labels_dict[l] for _ in range(len(image_paths))]

            elif e == 1:
                p = dir_paths[e] + "/" + l + "/"
                image_paths = [p + f for f in listdir(p) if isfile(p + f)]

                for img_p in image_paths:
                    temp = np.asarray(Image.open(img_p))
                    X_val.append(temp)
                y_val = y_val + [labels_dict[l] for _ in range(len(image_paths))]

    return np.array(X_train), to_categorical(np.array(y_train).reshape(-1, 1)), \
            np.array(X_val), to_categorical(np.array(y_val).reshape(-1, 1))

def main(verbose=False):
    # Paths
    DIR = "processed"
    train_dir_path = f"../data/{DIR}/train"
    val_dir_path = f"../data/{DIR}/val"

    # Loading Images
    X_train, y_train, X_val, y_val = load_images(train_dir_path, val_dir_path)

    if verbose:
        print("n_elements of: X_train - {},  y_train - {}".format(len(X_train), len(y_train)))
        print("n_elements of: X_val - {},  y_val - {}".format(len(X_val), len(y_val)))
        print(y_train)

    return X_train, y_train, X_val, y_val

if __name__ == '__main__':
    main(verbose=True)

max_stars_repo_name: MikeLrUC/IA-Project

max_issues_repo_name: MikeLrUC/IA-Project

max_forks_repo_name: MikeLrUC/IA-Project


text:
import cv2
import numpy as np
import matplotlib.pyplot as plt
import random

def sp_noise(image,prob):
    '''
    Add salt and pepper noise to image
    prob: Probability of the noise
    '''
    output = np.zeros(image.shape,np.uint8)
    thres = 1 - prob
    for i in range(image.shape[0]):
        for j in range(image.shape[1]):
            rdn = random.random()
            if rdn < prob:
                output[i][j] = 0
            elif rdn > thres:
                output[i][j] = 255
            else:
                output[i][j] = image[i][j]
    return output

def convolve_median(img):
    m, n = img.shape
    img_new = np.zeros([m, n])
    for i in range(1, m-1):
        for j in range(1, n-1):
            temp = [img[i-1, j-1],
                   img[i-1, j],
                   img[i-1, j + 1],
                   img[i, j-1],
                   img[i, j],
                   img[i, j + 1],
                   img[i + 1, j-1],
                   img[i + 1, j],
                   img[i + 1, j + 1]]

            temp = sorted(temp)
            img_new[i, j]= temp[4]
    img_new = img_new.astype(np.uint8)
    return img_new

img = cv2.imread('../data/pokemon.jpg')
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
noise_img = sp_noise(img,0.05)
convoleMedianOutput = convolve_median(noise_img)

plt.figure(figsize=(20,15))
plt.subplot(1,2,1),plt.imshow(noise_img,cmap = 'gray');
plt.title('Noise')
plt.subplot(1,2,2),plt.imshow(convoleMedianOutput,cmap = 'gray');
plt.title('Median Filter')

max_stars_repo_path: Order Filters/medianFilter.py

max_issues_repo_path: Order Filters/medianFilter.py

max_forks_repo_path: Order Filters/medianFilter.py


text:
'''
MIT License

Copyright (c) 2021
Michal Adamik, Jozef Goga, Jarmila Pavlovicova, Andrej Babinec, Ivan Sekaj

Faculty of Electrical Engineering and Information Technology
of the Slovak University of Technology in Bratislava
Ilkovicova 3, 812 19 Bratislava 1, Slovak Republic

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
'''

import numpy as np
from PIL import Image
from PIL import ImageDraw
from image4layer import Image4Layer

# ----------------- DEFINITIONS ------------------
COLOUR_BLACK = (0, 0, 0, 255) # black background colour
COLOUR_WHITE = (255, 255, 255, 255) # white background colour

# ----------------- FUNCTIONS  ------------------
def selbest(Oldpop,Fvpop,Nums):
    """
    The function copies best chromosomes from the old population into the new
    population required number of strings according to their fitness. The
    number of the selected chromosomes depends on the input vector Nums.
    The best chromosome is the chromosome with the lowest value of its
    objective function.

    Args:

        Oldpop: The primary population

        Fvpop: The fitness vector of primary population (Oldpop)

        Nums:  Vector in the form: Nums=[number of copies of the best chromosome, ... ,
                                         number of copies of the i-th best chromosome, ...]

    Returns:

        Newpop: The selected population based on fitness and specified input vector Nums

        Newfit: The fitness vector of newly created population (Newpop)

    """

    if(len(Fvpop.shape)<=1):
        Fvpop = np.reshape(Fvpop, (1,len(Fvpop)))
    N = len(Nums)
    fit = np.sort(Fvpop)
    nix = np.argsort(Fvpop)
    Newpop0 = np.zeros((N, Oldpop.shape[1]))
    Newpop = np.zeros((int(np.sum(Nums)), Oldpop.shape[1]))
    Newfit = np.zeros((int(np.sum(Nums),)))

    for i in range(N):
        Newpop0[i,:] = Oldpop[nix[0,i],:]

    r = 0
    for i in range(N):
        for j in range(Nums[i]):
            Newpop[r,:] = Newpop0[i,:]
            Newfit[r] = fit[0,i]
            r += 1

    return [Newpop, Newfit]

def selsus(Oldpop,Fvpop,n):
    """
    The function selects from the old population a required number of
    chromosomes using the "stochastic universal sampling" method. Under this
    selection method the number of a parent copies in the selected new
    population is proportional to its fitness.

    Args:

        Oldpop: The primary population

        Fvpop: The fitness vector of primary population (Oldpop)

        n:  Required number of selected chromosomes

    Returns:

        Newpop: The selected population based on stochastic universal sampling

        Newfit: The fitness vector of newly created population (Newpop)

    """

    if(len(Fvpop.shape)<=1):
        Fvpop = np.reshape(Fvpop, (1,len(Fvpop)))
    Newpop = np.zeros((n, Oldpop.shape[1]))
    OldFvpop = np.copy(Fvpop)
    Newfit = np.zeros((n,))
    lpop, lstring = Oldpop.shape
    Fvpop = Fvpop - np.min(Fvpop) + 1
    sumfv = np.sum(Fvpop).astype(np.float32)
    w0 = np.zeros((lpop+1,))

    for i in range(lpop):
        men = Fvpop[0,i]*sumfv
        w0[i] = 1.0/men # creation of inverse weights
    w0[i+1] = 0
    w = np.zeros((lpop+1,))
    for i in np.arange(lpop-1,-1,-1):
        w[i] = w[i+1] + w0[i]
    maxw = np.max(w)
    if (maxw==0):
        maxw = 0.00001
    w = (w/maxw)*100 # weigth vector
    pdel = 100.0/n
    b0 = np.random.uniform()*pdel - 0.00001
    b = np.zeros((n,))
    for i in range(1,n+1):
        b[i-1] = (i-1)*pdel + b0
    for i in range(n):
        for j in range(lpop):
            if(b[i]<w[j] and b[i]>w[j+1]):
                break
        Newpop[i,:] = Oldpop[j,:]
        Newfit[i] = OldFvpop[0,j]

    return [Newpop,Newfit]

def genLinespop(popsize, Amps, Space):
    """
    The function generates a population of random real-coded chromosomes which
    genes are limited by a two-row matrix Space. The first row of the matrix
    Space consists of the lower limits and the second row consists of the upper
    limits of the possible values of genes representing coordinates of line
    segments. The endpoints are generated by addition or substraction of
    random real-numbers to the mutated genes. The absolute values of the added
    values are limited by the vector Amp.

    Args:

        popsize: The size of the population (number of chromosomes to be created)

        Amps: Matrix of endpoints generation boundaries in the form:
    			[real-number vector of lower limits;
                 real-number vector of upper limits];

        Space:  Matrix of gene boundaries in the form:
	                [real-number vector of lower limits of genes;
                     real-number vector of upper limits of genes];

    Returns:

        Newpop: The created population of line segments inside
                specified boudaries

    """

    lpop, lstring = Space.shape
    Newpop = np.zeros((int(popsize),int(lstring)))
    if(len(Amps.shape)<=1):
        Amps = np.reshape(Amps, (1,len(Amps)))

    for r in range(int(popsize)):
        dX = Space[1,0] - Space[0,0]
        dY = Space[1,2] - Space[0,2]
        Newpop[r,0] = np.random.uniform()*dX + Space[0,0]
        Newpop[r,2] = np.random.uniform()*dY + Space[0,2]
        for s in [0,2]:
            if(Newpop[r,s]<Space[0,s]):
                Newpop[r,s] = Space[0,s]
            if(Newpop[r,s]>Space[1,s]):
                Newpop[r,s] = Space[1,s]

        Newpop[r,1] = Newpop[r,0] + np.random.randint(2*Amps[0,1]+1) - Amps[0,1]
        Newpop[r,3] = Newpop[r,2] + np.random.randint(2*Amps[0,3]+1) - Amps[0,3]

        for s in [1,3]:
            if(Newpop[r,s]<Space[0,s]):
                Newpop[r,s] = Space[0,s]
            if(Newpop[r,s]>Space[1,s]):
                Newpop[r,s] = Space[1,s]

    return Newpop

def mutLine(Oldpop,factor,Amps,Space):
    """
    The function mutates the population of chromosomes with the intensity
	proportional to the parameter rate from interval <0;1>. Only a few genes
	from a few chromosomes are mutated in the population. The mutations are
    realized by addition or substraction of random real-numbers to the mutated
    genes. The absolute values of the added constants are limited by the vector
    Amp. Next the mutated strings are limited using boundaries defined in
	a two-row matrix Space. The first row of the matrix represents the lower
	boundaries and the second row represents the upper boundaries of
    corresponding genes.

    Args:

        Oldpop: The primary population

        factor: The mutation rate, 0 =< rate =< 1

        Amps: Matrix of gene mutation boundaries in the form:
    			[real-number vector of lower limits;
                 real-number vector of upper limits];

        Space:  Matrix of gene boundaries in the form:
	                [real-number vector of lower limits of genes;
                     real-number vector of upper limits of genes];

    Returns:

        Newpop: The mutated population

    """

    if(len(Amps.shape)<=1):
        Amps = np.reshape(Amps, (1,len(Amps)))
    lpop, lstring = Oldpop.shape

    if (factor>1):
        factor = 1
    if (factor<0):
        factor = 0

    n = int(np.ceil(lpop*lstring*factor*np.random.uniform()))
    Newpop = np.copy(Oldpop)

    for i in range(n):
        rN = np.random.randint(2)
        r = int(np.ceil(np.random.uniform()*lpop))-1
        if (rN==0): # x1, y1
            Newpop[r,0] = Oldpop[r,0] + (2.0*np.random.uniform()-1)*Amps[0,0]
            Newpop[r,2] = Oldpop[r,2] + (2.0*np.random.uniform()-1)*Amps[0,2]
        elif (rN==1): # x2, y2
            Newpop[r,1] = Oldpop[r,1] + (2.0*np.random.uniform()-1)*Amps[0,1]
            Newpop[r,3] = Oldpop[r,3] + (2.0*np.random.uniform()-1)*Amps[0,3]

        for s in range(4):
            if (Newpop[r,s]<Space[0,s]):
                Newpop[r,s] = Space[0,s]
            if (Newpop[r,s]>Space[1,s]):
                Newpop[r,s] = Space[1,s]

    return Newpop

def evalFitness(Pop, orimg, geimg, fitOld, LINE_WIDTH, BLEND_MODE):
    """
    Sequentially evaluate fitness function for each chromosome in the population.

    Args:

        Pop: The population to be evaluated

        orimg: Template image

        geimg: The current generated image

        fitOld: The last fitness vector (from previous generation)

        LINE_WIDTH: The width of the line segments (in pixels)

        BLEND_MODE: Mode for blending two image layers

    Returns:

        Fit: The computed fitness vector

    """

    if(len(Pop.shape)<=1):
        Pop = np.reshape(Pop, (1,len(Pop)))
    lpop, lstring = Pop.shape
    Fit = np.zeros((lpop,))

    for i in range(lpop):
        G = Pop[i,:]
        Fit[i] = evalPartialSimilarity(list([orimg, geimg, G, fitOld, LINE_WIDTH, BLEND_MODE]))
    return Fit

def evalPartialSimilarity(imglist):
    """
    Evaluate partial similarity between original and generated image.

    Args:

        imglist: list of input parameters

            orimg: Template image

            geimg: The current generated image

            gen: The genetic information of current chromosome (coordinates)

            prevFit: The last value of the fitness function (from previous generation)

            LINE_WIDTH: The width of the line segments (in pixels)

            BLEND_MODE: Mode for blending two image layers

    Returns:

        The updated value of the fitness function

    """

    orimg = imglist[0]
    geimg = imglist[1]
    gen = imglist[2]
    prevFit = imglist[3]
    LINE_WIDTH = imglist[4]
    BLEND_MODE = imglist[5]

    # evaluate only part of the image
    minX = int(np.min(gen[0:2]))
    maxX = int(np.max(gen[0:2]))
    deltaX = int(maxX - minX) + 1

    minY = int(np.min(gen[2:4]))
    maxY = int(np.max(gen[2:4]))
    deltaY = int(maxY - minY) + 1

    # first evaluation
    if prevFit is None:
        # create a blank canvas
        draw = Image.new('RGBA', (deltaY, deltaX), (255,255,255,0))
        draw = draw.convert('L')
        pdraw = ImageDraw.Draw(draw)
        line = ((int(gen[2])-minY, int(gen[0])-minX),(int(gen[3])-minY, int(gen[1])-minX))
        mask_img = Image.new('1', (draw.size[0], draw.size[1]), 0)
        ImageDraw.Draw(mask_img).line(line, fill=1, width=LINE_WIDTH)
        mask = np.array(mask_img)

        tgrey = orimg[minX:minX+deltaX, minY:minY+deltaY] * mask
        tgrey = tgrey[tgrey != 0]

        # compute the lighest shade of the line segment
        c = int(np.max(tgrey))

        # create new line segment
        pdraw.line(line, fill=(c), width=LINE_WIDTH)
        partgenImg = geimg.crop((minY, minX, minY + deltaY, minX + deltaX))

        # call blending mode function by name
        out = eval('Image4Layer.' + BLEND_MODE)(draw, partgenImg)

        # evaluate full image
        gout = Image.new('RGBA', geimg.size, COLOUR_WHITE)
        gout = gout.convert('L')
        gout.paste(out, (minY, minX))

        # compute similarity between original and generated image
        return np.sqrt(np.sum((orimg - np.asarray(gout, dtype=int))**2.0)/(gout.size[0]*gout.size[1])) / 255.0

    else:
        # reconstruct previous similarity
        tSum = ((prevFit*255.0)**2.0) * geimg.size[1] * geimg.size[0]

        # create a blank canvas
        draw = Image.new('RGBA', (deltaY, deltaX), (255,255,255,0))
        draw = draw.convert('L')
        pdraw = ImageDraw.Draw(draw)
        line = ((int(gen[2])-minY, int(gen[0])-minX),(int(gen[3])-minY, int(gen[1])-minX))
        mask_img = Image.new('1', (draw.size[0], draw.size[1]), 0)
        ImageDraw.Draw(mask_img).line(line, fill=1, width=LINE_WIDTH)
        mask = np.array(mask_img)

        tgrey = orimg[minX:minX+deltaX, minY:minY+deltaY] * mask
        tgrey = tgrey[tgrey != 0]

        # if mask is an empty array
        if (tgrey.size == 0):
            return prevFit

        # compute the lightest shade of the line segment
        c = int(np.max(tgrey))

        # create new line segment
        pdraw.line(line, fill=(c), width=LINE_WIDTH)
        partgenImg = geimg.crop((minY, minX, minY + deltaY, minX + deltaX))

        # call blending mode function by name
        out = eval('Image4Layer.' + BLEND_MODE)(draw, partgenImg)

        # substract similarity between previously generated image and target image + add newly computed part
        newSum = tSum - np.sum((orimg[minX:minX+deltaX, minY:minY+deltaY] - np.asarray(partgenImg, dtype=int))**2.0) + np.sum((orimg[minX:minX+deltaX, minY:minY+deltaY] - np.asarray(out, dtype=int))**2.0)
        if (newSum < 0):
            return prevFit
        else:
            return np.sqrt(newSum/(geimg.size[0]*geimg.size[1])) / 255.0

max_stars_repo_name: xgoga/FastRoboticPencilDrawing

max_issues_repo_name: xgoga/FastRoboticPencilDrawing

max_forks_repo_name: xgoga/FastRoboticPencilDrawing


text:
import tensorflow as tf
import numpy as np
import math
from tensorflow.python.ops import variable_scope
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import nn_ops
from tensorflow.python.ops import math_ops
import nn_lib

# 带有embedding的RNN模型
def rnn_nlp(x, keep_prob, word_embd_pretrain, dim_rnn, dim_y):
    with tf.variable_scope('rnn_nlp'):
        # layer[0] 输入数据，未embedding前
        # x:[batch,step]
        layer_output = [x]
        # layer[1] 进行embedding
        # embedding_w2v:[batch,step,embd]
        embedding_w2v = tf.nn.embedding_lookup(word_embd_pretrain, layer_output[0])
        layer_output.append(embedding_w2v)
        # layer[2] 生成lstm实体，并进行dropout
        lstm_cell_raw = tf.nn.rnn_cell.BasicLSTMCell(num_units=dim_rnn, state_is_tuple=True)
        lstm_cell = tf.nn.rnn_cell.DropoutWrapper(cell=lstm_cell_raw, output_keep_prob=keep_prob)
        lstm_layer = tf.nn.dynamic_rnn(lstm_cell, layer_output[1], dtype=tf.float32)[0]
        layer_output.append(lstm_layer)
        # layer[3]
        # transpose:[step,batch,embd]
        transpose = tf.transpose(layer_output[2], [1, 0, 2])
        # transpose_gather:[batch,embd]提取lstm最后一个时刻的output数据
        transpose_gather = tf.gather(transpose, int(transpose.shape[0].value)-1)
        layer_output.append(transpose_gather)
        # layer[4] 全连接层
        weight = tf.get_variable(name='weight', trainable=True,
                    initializer=tf.truncated_normal([dim_rnn, dim_y], stddev=math.sqrt(6/(dim_rnn+dim_y)), dtype=tf.float32))
        bias = tf.get_variable(name='bias', trainable=True,
                               initializer=tf.truncated_normal([dim_y], stddev=0.1, dtype=tf.float32))
        layer_output.append((tf.matmul(layer_output[3], weight) + bias))
    return layer_output

max_stars_repo_name: archfool/nlp

max_issues_repo_name: archfool/nlp

max_forks_repo_name: archfool/nlp


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Demo showing how km_dict and insegtannotator may be used together for
interactive segmentation.

@author: vand and abda
"""

import sys
import insegtannotator
import skimage.io
import skimage.data
import km_dict
import numpy as np

#%% EXAMPLE 1: glass fibres

## loading image
print('Loading image')
filename = '../data/glass.png'
image = skimage.io.imread(filename)

#%% EXAMPLE 2: nerve fibres

## loading image
print('Loading image')
filename = '../data/nerve_im_scale.png'
image = skimage.io.imread(filename)

#%% COMMON PART

patch_size = 11
branching_factor = 5
number_layers = 5
number_training_patches = 35000
normalization = False

image_float = image.astype(np.float)/255

# Build tree
T = km_dict.build_km_tree(image_float, patch_size, branching_factor, number_training_patches, number_layers, normalization)
# Search km-tree and get assignment
A, number_nodes = km_dict.search_km_tree(image_float, T, branching_factor, normalization)
# number of repetitions for updating the segmentation
number_repetitions = 2

def processing_function(labels):
    r,c = labels.shape
    l = np.max(labels)+1
    label_image = np.zeros((r,c,l))
    for k in range(number_repetitions):
        for i in range(1,l):
            label_image[:,:,i] = (labels == i).astype(float)
        D = km_dict.improb_to_dictprob(A, label_image, number_nodes, patch_size) # Dictionary
        P = km_dict.dictprob_to_improb(A, D, patch_size) # Probability map
        labels = np.argmax(P,axis=2) # Segmentation
    return labels

print('Showtime')

# showtime
app = insegtannotator.PyQt5.QtWidgets.QApplication([])
ex = insegtannotator.InSegtAnnotator(image, processing_function)
app.exec()
sys.exit()

max_stars_repo_name: Skielex/InSegt

max_issues_repo_name: Skielex/InSegt

max_forks_repo_name: Skielex/InSegt


text:
"""
Contains common functions to arim.scat and arim.model, to avoid always importing
arim.scat from arim.model.
"""
import numba
import numpy as np

@numba.jit(nopython=True, cache=True)
def _interpolate_scattering_matrix_kernel(scattering_matrix, inc_theta, out_theta):
    # This is a kernel which takes one incident angle and one scattered angle.
    numpoints = scattering_matrix.shape[0]
    dtheta = 2 * np.pi / numpoints

    # Returns indices in [0, ..., numpoints - 1]
    # -pi <-> 0
    # pi - eps <-> numpoints - 1
    # -pi + 2 k pi <-> 0
    inc_theta_idx = int((inc_theta + np.pi) // dtheta % numpoints)
    out_theta_idx = int((out_theta + np.pi) // dtheta % numpoints)

    # Returns the fraction in [0., 1.[ of the distance to the next point to the distance
    # to the last point.
    inc_theta_frac = ((inc_theta + np.pi) % dtheta) / dtheta
    out_theta_frac = ((out_theta + np.pi) % dtheta) / dtheta

    # if we are on the border, wrap around (360° = 0°)
    if inc_theta_idx != (numpoints - 1):
        inc_theta_idx_plus1 = inc_theta_idx + 1
    else:
        inc_theta_idx_plus1 = 0

    if out_theta_idx != (numpoints - 1):
        out_theta_idx_plus1 = out_theta_idx + 1
    else:
        out_theta_idx_plus1 = 0

    # use cardinal direction: sw for south west, etc
    sw = scattering_matrix[out_theta_idx, inc_theta_idx]
    ne = scattering_matrix[out_theta_idx_plus1, inc_theta_idx_plus1]
    se = scattering_matrix[out_theta_idx, inc_theta_idx_plus1]
    nw = scattering_matrix[out_theta_idx_plus1, inc_theta_idx]

    # https://en.wikipedia.org/wiki/Bilinear_interpolation
    f1 = sw + (se - sw) * inc_theta_frac
    f2 = nw + (ne - nw) * inc_theta_frac
    return f1 + (f2 - f1) * out_theta_frac

@numba.guvectorize(
    ["void(f8[:,:], f8[:], f8[:], f8[:])", "void(c16[:,:], f8[:], f8[:], c16[:])"],
    "(s,s),(),()->()",
    nopython=True,
    target="parallel",
    cache=True,
)
def _interpolate_scattering_matrix_ufunc(scattering_matrix, inc_theta, out_theta, res):
    res[0] = _interpolate_scattering_matrix_kernel(
        scattering_matrix, inc_theta[0], out_theta[0]
    )

max_stars_repo_name: will-jj/arim

max_issues_repo_name: will-jj/arim

max_forks_repo_name: will-jj/arim

reason: import numpy,import numba


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Dec  2 18:52:36 2019

@author: cdebezenac
"""

import numpy as np
from shapely.geometry import Point,MultiPoint,Polygon
from shapely.ops import cascaded_union
import math
import pandas as pd
import geopandas as gdp
from divergence.basic_analysis import Demographics

def binarize(matrix,threshold):
    '''Transform numeric matrix in binary form: value > 1
    Parameter
    ---------
    - matrix: numeric numpy array
        matrix of numeric value to binarize based on the rule value < threshold
    - threshold: numeric (float, int)
         Threshold for True or False values in matrix.

    Returns
    -------
    binarized matrix with value 0 if matrix[i,j]<threshold, else 1.
    '''
    return((matrix>=threshold)*1)

def calculate_distances(coordinates):
    '''Computes the distance matrix representing the crow distance between centroids of polygons.
    Parameters
    ----------
    coordinates: numpy array, GeoSeries of Point type
        list of centroid Point coordinates in neighbouring spatial units.

    Returns
    -------
    distance matrix : sqaure numpy array of floats
        matrix of euclidean distances between centroids of Polygons.
    '''
    #initialise an 'empty' matrix
    distance_matrix=np.zeros((len(coordinates),len(coordinates)))
    for i in range(len(coordinates)):
        for j in range (i,len(coordinates)):
            #calculate the crow distance between centroids of i and j
            distance_matrix[i,j] = coordinates[i].distance(coordinates[j])
            #crow distance being a metric, the matrix is symmetric
            distance_matrix[j,i] =  distance_matrix[i,j]
    return(distance_matrix)

def calculate_demographic_dissimilarity(population_matrix):
    '''Computes the dissimilarity matrix representing the difference in population count for each group in each unit.
    Parameters
    ----------
    population_matrix: numpy array of integers
        population count for categorial groups in each spatial unit.

    Returns
    -------
    demographic type dissimilarity matrix: sqaure numpy array
        matrix of euclidean distance between population proportions in any two units included in the input matrix.
    '''
    #calculate the total population count in each unit
    sum_pop_local=np.sum(population_matrix,axis=1)
    #calculate each group's local proportion in each unit.
    local_proportion=population_matrix/sum_pop_local[:,np.newaxis]
    #initialise an 'empty' square matrix
    demo_matrix=np.zeros((len(population_matrix),len(population_matrix)))

    for i in range(len(population_matrix)):
        for j in range (i,len(population_matrix)):
            #for each couple of units (i,j), compute the euclidean distance of population count
            demo_matrix[i,j] = np.sqrt(np.sum((local_proportion[i]-local_proportion[j])**2))
            #symmetric metric
            demo_matrix[j,i] =  demo_matrix[i,j]
    return(demo_matrix)

def calculate_divergence_dissimilarity(population_matrix):
    '''Computes the KL divergence matrix representing the KL divergence between any two units in the city.
    Parameters
    ----------
    population_matrix: numpy array of integers
        population count for categorial groups in each spatial unit.

    Returns
    -------
    divergence type dissimilarity matrix: sqaure numpy array
        matrix of KL divergence between any two units included in the input matrix, based on population distribution.
    '''
    #check for null values of population and make them very small instead, to avoid KL divergence from going to infinity:
    pop_matrix=np.where(population_matrix==0, 0.0001, population_matrix)
    #calculate the total population count in each unit
    sum_pop_local=np.sum(pop_matrix,axis=1)
    #calculate each group's local proportion in each unit
    local_proportion=pop_matrix/sum_pop_local[:,np.newaxis]
    #initialise square matrix:
    divergence_matrix=np.zeros((len(pop_matrix),len(pop_matrix)))

    for unit1 in range(len(pop_matrix)):
        for unit2 in range (len(pop_matrix)):
            #go through all couples twice because KL divergence is NOT symmetric
            #calculate the ratio of local proportion on global proportions for each group in each unit.
            relative_prop=local_proportion[unit1]/local_proportion[unit2]
            #calculate the log of the ratio
            log_relative = np.log(relative_prop)
            #check for exceptions and turn 0log0 values to 0
            log_relative[log_relative == -math.inf] = 0
            #fill in the divergence matrix with value
            divergence_matrix[unit1,unit2]=np.sum(np.multiply(local_proportion[unit1],log_relative))
    return(divergence_matrix)

def is_adjacent(coordinates):
    '''Computes the adjacent matrix for the spatial units in the city.
    Parameters
    ----------
    coordinates: GeoSeries of Polygons
        coordinates of spatial unit outile.

    Returns
    -------
    adjacent matrix: binary matrix of 0 or 1 if units touch in the "rook" sense, sharing a border.
    '''
    adjacent=np.zeros((len(coordinates),len(coordinates)))
    for i in range(len(coordinates)):
        #use is_touching() to check if units are adjacent
        is_touching=coordinates.touches(coordinates[i])
        #symmetrix relation:
        adjacent[i,:] = is_touching
        #add relation to itself (neighbours to itself)
        adjacent[i,i]=1
    return(adjacent)

class Neighbourhood():
    '''
    Builds the neighbourhood structure of spatial data unit based on the type of neighbourhood selected and the data attributes.
    This neighbourhood represents more specifically all the spatial units in the city ordered according to a similarity rule,
    from closest to farthest of a given unit. Without specifying a particular definition for neighborhood,
    this structure will include the entire city at all possible scales. The algorithm is repeated for all possible origins.

    Parameters
    ==========
        - geodata: Geopandas
                   Dataframe to analyse with geometry columns.
        - groups: list of string
                  list of names of categorical variables to analyse (ethnic groups for instance).
        - method: string
                  name of method to calculate dissimilarity between units.

    Attributes
    ==========
        - coordinates: GeoSeries Polygon shape
                       coordinates of spatial units in data.
        - population_matrix: numpy array of integers ; shape (len(data),len(groups))
                             population count for all groups in all units.
        - adjacency_matrix: square numpy array
                            binary matrix of touching neighbours (rook neighbourhood).
        - distance_matrix: square numpy array
                           symmetric matrix representing crow distances between all units.
        - dissimilarity_matrix: square numpy array
                             non-symmetric or symmetric matrix representing a dimension of dissimilarity,
                             bilateral KL divergence between units or demographic difference in population counts.
        - neighbours: square numpy array of Index class
                      indexes of ordered neighbours for each unit.

    Methods
    =======
        - set_euclidean_neighbours()
            Returns: - distance_matrix: numpy array floats.
                       euclidean distances between units
                     - neighbours: numpy array of indices
                       ordered list of units from smallest to largest crow distance for each unit

        - set_neighbourhood_type()

    Example
    =======

    '''

    def __init__(self,geodata,groups):
        #access coordinates of all Polygons in city
        self.coordinates=geodata.geometry
        #create a Demographic object and access the population_matrix (does not keep in memory the rest of the data)
        #(This step seems redundant with DivergenceProfile.demography and could be skipped if the city Demographic attribute was accessed directly, but this enables users to create neighbourhood structures independantly of the segregation analysis).
        self.population_matrix=Demographics(geodata,groups).population_matrix
        #calculate the adjacecny matrix for coty coordinates.
        self.adjacency_matrix=is_adjacent(self.coordinates)

    def set_euclidean_neighbours(self):
        '''
        Call this method to initialise a neighbourhood structure based on the crow distance between unit centroids.

        Returns
        -------
        - distance_matrix: numpy array floats
            euclidean distances between units
        - neighbours: numpy array of indices
            ordered list of units from smallest to largest crow distance for each unit

        '''
        self.distance_matrix=calculate_distances(self.coordinates.centroid)
        self.neighbours=np.argsort(self.distance_matrix)

    def set_neighbourhood_type(self,method='demographic'):
        '''
        Call this method to create a hybrid neighbourhood structure based on adjacency and similarity relating to a particular variable specified in *method*.
        Parameters
        ----------
        method : string
            specific type of neighbourhood structure. Multiple choice in ['demographic','divergence']
                * 'demographic': the closest neighbour touches the origin and has the most similar population proportions (and so on).
                * 'divergence: the closest neighbour touches the origin and has the smallest bilateral KL divergence with the origin.
            The *divergence* method is a non-linear transformation of the *demographic* method.
        Returns
        -------
        - dissimilarity_matrix: numpy array of floats ; shape(len(coordinates)**2)
            dissimilarity between units given a distance or divergence method.
        - neighbours : numpy array of indices ; shape(len(coordinates)**2)
            neighbourhood based on dissimilarity (divergence or demographic distance) and adjacency.'''

        if method=='demographic':
            #call the particular Neighbourhood method relating the the parameter method to compute the square dissimilarity matrix
            self.dissimilarity_matrix=calculate_demographic_dissimilarity(self.population_matrix)
        elif method=='divergence':
            self.dissimilarity_matrix=calculate_divergence_dissimilarity(self.population_matrix)
        else :
            raise ValueError('the '+method+' is unknown. Please choose between "euclidean", "divergence" or "demographic".')

        #access the maximum dissimilarity between any two units
        max_diff=self.dissimilarity_matrix.max()
        #initialise the list of matrices of incresaing adjacency degree : first, second, thrid, etc... degree neighbours.
        adjacent_structure=[self.adjacency_matrix]
        #calculate the first degree dissimilarity matrix (dissimilarity between first degree neighbours)
        adjacent_difference=self.adjacency_matrix * self.dissimilarity_matrix
        #initialise degree of neighbourhood k
        k=1
        #initialise the adjacency matrix of degree k
        next_adjacent=1
        #include a stopping condition for the algorithm : that there are no more neighbours left to visit.
        while np.sum(next_adjacent)>0:
            #multiply the k-1 degree matrix by the original adjacency matrix: this will give the k degree of neighbourhood: if b is a's neighbour and c is b's, then the next degree neighbour of a will include c.
            k_neighbourhood=adjacent_structure[-1].dot(self.adjacency_matrix)
            #get the adjacent neighbors of degree k by excluding all existing ones already : use binarize() to set to 0 and 1 and subtract all the previous adjacency matrices from inferior neighbourhoo degree.
            next_adjacent=binarize(binarize(k_neighbourhood,1) - sum(adjacent_structure),1)
            adjacent_structure.append(next_adjacent)
            #calculate the dissimilarity matrix of degree k and add the maximum of dissimilarity multiplied by the neighbourhood degree, max_diff, to differentiate between degrees of neighbourhood once all dissimilarity matrices joint into one matrix.
            adjacent_difference += next_adjacent * (self.dissimilarity_matrix + (max_diff+1) * k)
            k+=1
        #sort by increasing order the rows of the matrix, this will order them first by degree of neighbourhood, then by smallest dissimilarity value.
        #this method avoids creating a dataframe for each unit with ordered degree and ordered dissimilarity but deals with all neighbours for all units in one single numpy array, which is quicker.
        self.neighbours=np.argsort(adjacent_difference)


text:
# Copyright 2017–2019 Forschungszentrum Jülich GmbH
# Copyright 2019–2020 CEA
#
# Author: Yann Leprince <yann.leprince@cea.fr>
#
# Licensed under the Apache Licence, Version 2.0 (the "Licence");
# you may not use this file except in compliance with the Licence.
# You may obtain a copy of the Licence at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the Licence is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the Licence for the specific language governing permissions and
# limitations under the Licence.

import logging

import numpy
import pytest

# Request sent by the original landmark-reg (AngularJS-based prototype)
TEST_LANDMARK_PAIRS = [
    {
        'active': True,
        'colour': '#8dd3c7',
        'name': 'most medial, inferior point',
        'source_point': [11.4, 8.6, 4.1],
        'target_point': [63.1, 74.7, 47.7],
    },
    {
        'active': True,
        'colour': '#ffffb3',
        'name': 'most lateral, superior point',
        'source_point': [4.1, 4.5, 4.1],
        'target_point': [56.5, 74.7, 52.4],
    },
    {
        'active': True,
        'colour': '#bebada',
        'name': 'most anterior point',
        'source_point': [10.224, 6.752, 9.74],
        'target_point': [62.2, 80.0, 49.5],
    },
    {
        'active': True,
        'colour': '#000000',
        'name': 'other nonsensical point',
        'source_point': [0.0, 1.0, 2.0],
        'target_point': [50.0, 51.0, 52.0],
    },
]

def test_least_squares_full_request(client):
    response = client.post('/api/least-squares', json={
        'landmark_pairs': TEST_LANDMARK_PAIRS,
        'source_image': 'http://h.test/source-image',
        'target_image': 'http://h.test/target-image',
        'transformation_type': 'similarity',
    })
    assert response.status_code == 200

    assert 'transformation_matrix' in response.json
    assert 'inverse_matrix' in response.json
    matrix = numpy.array(response.json['transformation_matrix'])
    inverse_matrix = numpy.array(response.json['inverse_matrix'])
    assert matrix.shape == (4, 4)
    assert inverse_matrix.shape == (4, 4)
    assert numpy.allclose(inverse_matrix @ matrix, numpy.eye(4))
    assert numpy.allclose(matrix @ inverse_matrix, numpy.eye(4))

    assert 'RMSE' in response.json
    assert isinstance(response.json['RMSE'], float)

    assert 'landmark_pairs' in response.json
    assert len(response.json['landmark_pairs']) == len(TEST_LANDMARK_PAIRS)
    for returned_pair, orig_pair in zip(response.json['landmark_pairs'],
                                        TEST_LANDMARK_PAIRS):
        assert 'source_point' in returned_pair
        assert numpy.allclose(returned_pair['source_point'],
                              orig_pair['source_point'])
        assert 'target_point' in returned_pair
        assert numpy.allclose(returned_pair['target_point'],
                              orig_pair['target_point'])
        assert 'mismatch' in returned_pair
        assert isinstance(returned_pair['mismatch'], float)

def test_least_squares_minimal_request(client):
    response = client.post('/api/least-squares', json={
        'landmark_pairs': [
            {
                'source_point': pair['source_point'],
                'target_point': pair['target_point'],
            }
            for pair in TEST_LANDMARK_PAIRS[:3]
        ],
        'transformation_type': 'similarity',
    })
    assert response.status_code == 200

    assert 'transformation_matrix' in response.json
    assert 'inverse_matrix' in response.json
    assert 'RMSE' in response.json
    assert 'landmark_pairs' in response.json
    assert len(response.json['landmark_pairs']) == 3
    for returned_pair, orig_pair in zip(response.json['landmark_pairs'],
                                        TEST_LANDMARK_PAIRS):
        assert 'source_point' in returned_pair
        assert numpy.allclose(returned_pair['source_point'],
                              orig_pair['source_point'])
        assert 'target_point' in returned_pair
        assert numpy.allclose(returned_pair['target_point'],
                              orig_pair['target_point'])
        assert 'mismatch' in returned_pair
        assert isinstance(returned_pair['mismatch'], float)

@pytest.mark.parametrize(
    'transformation_type',
    [
        'rigid',
        'similarity',
        'rigid+reflection',
        'similarity+reflection',
        pytest.param('affine',
                     marks=pytest.mark.skip('affine estimation does not handle'
                                            'ill-conditioned cases yet')),
    ]
)
def test_least_squares_underconstrained(client, transformation_type):
    response = client.post('/api/least-squares', json={
        'landmark_pairs': [
            {
                'source_point': [0, 0, 0],
                'target_point': [0, 0, 0],
            },
            {
                'source_point': [1, 0, 0],
                'target_point': [1, 0, 0],
            },
        ],
        'transformation_type': transformation_type,
    })
    assert response.status_code == 400
    assert 'message' in response.json
    assert 'under' in response.json['message']

@pytest.mark.parametrize(
    ['transformation_type', 'point_count'],
    [
        ('rigid', 3),
        ('similarity', 3),
        ('rigid+reflection', 4),
        ('similarity+reflection', 4),
        ('affine', 4),
    ]
)
def test_least_squares_transformation_types(client,
                                            transformation_type,
                                            point_count):
    response = client.post('/api/least-squares', json={
        'landmark_pairs': TEST_LANDMARK_PAIRS[:point_count],
        'transformation_type': transformation_type,
    })
    assert response.status_code == 200

def test_least_squares_debug_log(client, caplog):
    caplog.set_level(logging.DEBUG)
    response = client.post('/api/least-squares', json={
        'landmark_pairs': TEST_LANDMARK_PAIRS,
        'transformation_type': 'affine',
    })
    assert 'least-squares' in caplog.text
    assert 'landmark_pairs' in caplog.text
    assert 'affine' in caplog.text
    assert response.status_code == 200

def test_least_squares_invalid_requests(client):
    response = client.get('/api/least-squares')
    assert response.status_code == 405

    response = client.post('/api/least-squares', json={
        'landmark_pairs': TEST_LANDMARK_PAIRS,
        'transformation_type': 'invalid',
    })
    assert 400 <= response.status_code < 500

    response = client.post('/api/least-squares', json={
        'transformation_type': 'invalid',
    })
    assert 400 <= response.status_code < 500

    response = client.post('/api/least-squares', json={
        'landmark_pairs': TEST_LANDMARK_PAIRS,
    })
    assert 400 <= response.status_code < 500

    response = client.post('/api/least-squares', json={})
    assert 400 <= response.status_code < 500

def test_transformation_matrix_field():
    import marshmallow
    from marshmallow.exceptions import ValidationError
    from linear_voluba.api import TransformationMatrixField

    class TestSchema(marshmallow.Schema):
        val = TransformationMatrixField(allow_none=True)

    schema = TestSchema()
    mat = schema.load({'val': [[1, 0, 0, 0],
                               [0, 1, 0, 0],
                               [0, 0, 1, 0],
                               [0, 0, 0, 1]]})
    assert isinstance(mat['val'], numpy.ndarray)
    assert numpy.array_equal(mat['val'], numpy.eye(4))
    mat = schema.load({'val': [[1, 0, 0, 0],
                               [0, 1, 0, 0],
                               [0, 0, 1, 0]]})
    assert isinstance(mat['val'], numpy.ndarray)
    assert numpy.array_equal(mat['val'], numpy.eye(4))
    mat = schema.load({'val': None})
    assert mat['val'] is None

    serialized = schema.dump({'val': numpy.eye(4)})
    assert serialized == {
        'val': [[1, 0, 0, 0],
                [0, 1, 0, 0],
                [0, 0, 1, 0],
                [0, 0, 0, 1]],
    }
    serialized = schema.dump({'val': None})
    assert serialized == {'val': None}

    with pytest.raises(ValidationError):
        schema.load({'val': 'string'})
    with pytest.raises(ValidationError):
        schema.load({'val': {'some': 'dict'}})
    with pytest.raises(ValidationError):
        schema.load({'val': 0})
    with pytest.raises(ValidationError):
        schema.load({'val': []})
    with pytest.raises(ValidationError):
        schema.load({'val': [[1, 2], [3]]})
    with pytest.raises(ValidationError):
        schema.load({'val': [[1, 0, 0, 0],
                             [0, 1, 0, 0],
                             [0, 0, 1, 0],
                             [1, 2, 3, 4]]})

    # Test the OpenAPI fields supplied as metadata. TODO: test actual
    # conversion to JSON Schema using apispec.
    assert schema.fields['val'].metadata['type'] == 'array'
    assert schema.fields['val'].metadata['minItems'] == 3
    assert schema.fields['val'].metadata['maxItems'] == 4
    assert schema.fields['val'].metadata['items'] == {
        "type": "array",
        "minItems": 4,
        "maxItems": 4,
        "items": {
            "format": "float",
            "type": "number"
        },
    }

max_stars_repo_name: HumanBrainProject/voluba-linear-backend

max_issues_repo_name: HumanBrainProject/voluba-linear-backend

max_forks_repo_name: HumanBrainProject/voluba-linear-backend