text:
import json

import requests
import streamlit as st

def run_the_app():

    st.sidebar.markdown("# Server location (URL)")

    url = st.sidebar.text_input("URL", "http://localhost:8000")

    st.sidebar.markdown("# Server authentication")

    auth = None
    user = st.sidebar.text_input("User name")
    password = st.sidebar.text_input("Password")
    if user and password:
        auth = (user, password)

    st.markdown("# Pathmind Policy Server")

    schema = server_schema(auth, url).json()
    print(schema)

    st.markdown("## Predictions")

    obs = generate_frontend_from_observations(schema)
    predict_button = st.button("Get prediction")
    if obs and auth and predict_button:
        response = predict(obs, auth, url).json()
        st.markdown(f"## Action: {response.get('actions')[0]}")
        st.markdown(f"## Probability: {int(100 *response.get('probability'))}%")

    # compute_action_distro = st.checkbox(label="What's the variance of my actions?", value=False)
    # if compute_action_distro and obs:
    #     distro_dict: dict = distro(obs, auth, url).json()
    #
    #     import matplotlib.pyplot as plt
    #     import numpy as np
    #     arr = np.asarray(list(distro_dict.values()))
    #     x_range = np.arange(len(distro_dict))
    #     plt.bar(x_range, arr)
    #     plt.xticks(x_range, list(distro_dict.keys()))
    #     st.pyplot(plt)

def server_schema(auth, url):
    return requests.get(f"{url}/schema", auth=auth)

def distro(observation, auth, url):
    return requests.post(f"{url}/distribution", json=observation, auth=auth)

def predict(observation, auth, url):
    return requests.post(f"{url}/predict/", json=observation, auth=auth)

def generate_frontend_from_observations(schema: dict):
    """Auto-generates front-end components from schema.

    :param schema: "Observation" schema loaded from YAML file
    :return: observation dictionary ready to be sent to server for processing.
    """
    properties = schema.get("observations")
    result = {}
    for key, values in properties.items():
        prop_type = values.get("type")
        # example = values.get("example")
        # description = values.get("description")
        if prop_type == "bool":
            val = st.checkbox(label=key, value=True)
        elif prop_type == "int":
            val = int(st.number_input(label=key, value=0))
            check_min_max(values, val)
        elif prop_type == "float":
            val = st.number_input(label=key, value=0.0)
            check_min_max(values, val)
        elif prop_type in ["List[int]", "List[float]", "List[bool]"]:
            val = json.loads(st.text_input(label=key, value=str([0])))
            validate_list_items(values, val)
            check_min_max_items(values, val)
        else:
            raise Exception(
                f"Unsupported data type {prop_type} in YAML schema for model server."
            )
        result[key] = val
    return result

def validate_list_items(values, val):
    items_type = values.get("type")
    if items_type == "integer":
        assert all(isinstance(v, int) for v in val), "All list items must be integers."
    elif items_type == "number":
        assert all(
            isinstance(v, float) for v in val
        ), "All list items must be floating point numbers."

def check_min_max_items(values, val):
    if "minItems" in values.keys():
        assert len(val) >= values.get(
            "minItems"
        ), f"Array too small, expected at least {values.get('minItems')} items."
    if "maxItems" in values.keys():
        assert len(val) <= values.get(
            "maxItems"
        ), f"Array too large, expected at most {values.get('maxItems')} items."

def check_min_max(values, val):
    if "minimum" in values.keys():
        assert val >= values.get(
            "minimum"
        ), f"value {val} too small, expected a minimum of {values.get('minimum')}."
    if "maximum" in values.keys():
        assert val <= values.get(
            "maximum"
        ), f"value {val} too large, expected a maximum of {values.get('maximum')}."

if __name__ == "__main__":
    run_the_app()

max_stars_repo_name: SkymindIO/policy-server

max_issues_repo_name: SkymindIO/policy-server

max_forks_repo_name: SkymindIO/policy-server


text:
import torch
import math
import numpy as np

DEFAULT_MU = torch.ones([4]) * 4
DEFAULT_SIGMA = torch.ones([4, 4]) * .1 + torch.eye(4)
DEFAULT_SIGMA_INV = torch.inverse(DEFAULT_SIGMA)

scale = 4
centers = [(1, 0), (-1, 0), (0, 1), (0, -1), (1. / np.sqrt(2), 1. / np.sqrt(2)),
                   (1. / np.sqrt(2), -1. / np.sqrt(2)), (-1. / np.sqrt(2),
                                                         1. / np.sqrt(2)), (-1. / np.sqrt(2), -1. / np.sqrt(2))]
centers = [(scale * x, scale * y) for x, y in centers]
DEFAULT_CENTERS = torch.tensor(centers)

# DEFAULT_CENTERS = torch.stack([torch.ones([4]) * i for i in range(8)])

def gaussian_score(z, mu=DEFAULT_MU, sigma=DEFAULT_SIGMA):
    mu = mu.to(z)
    sigma = sigma.to(z)

    assert mu.shape[-1] == sigma.shape[-1]

    sigma_inv = torch.inverse(sigma).to(z).unsqueeze(0).expand(z.shape[0], -1, -1)
    score = torch.matmul(sigma_inv, -z.unsqueeze(2) + mu.unsqueeze(1))
    return score.squeeze()

class IsometricGaussianMollifier(object):
    def __init__(self, sigma_square=5e-2):
        self.sigma_square = sigma_square

    def score(self, centers, particles):
        '''
        This function computes the score of the Gaussian mixture at particles.
        The centers of the Gaussian mixture is given in centers. The variance of the Gaussians are self.sigma_square
        Args:
            centers: centers of the Gaussian mixture
            particles: position of the particles

        Returns: score of the Gaussian mixture at particles

        '''
        z = particles.unsqueeze(dim=1).expand(-1, centers.shape[0], -1) - centers
        b = torch.sum(z.pow(2), dim=2) / 2 / self.sigma_square
        a = torch.exp(-b) + 1e-12
        normalized_a = (a / a.sum(dim=1, keepdim=True)).unsqueeze(dim=2)

        # c = -z+centers
        score = torch.sum(-z * normalized_a/self.sigma_square, dim=1)

        # print(torch.norm(score - score_2))
        return score

def gaussian_mixture_score(z, centers):

    centers = torch.tensor(centers).to(z)

    z = z.unsqueeze(dim=1).expand(-1, centers.shape[0], -1) - centers
    a = torch.sum(z.pow(2), dim=2)/2.
    a = torch.exp(-a)
    normalized_a = (a/a.sum(dim=1, keepdim=True)).unsqueeze(dim=2)
    # c = -z+centers
    score = torch.sum(-z*normalized_a, dim=1)

    # print(torch.norm(score - score_2))
    return score

def gaussian_mixture_logprob(z):
    # centers = torch.tensor([(4, 4), (4, 3), (4, 2)]).to(z)
    # centers = torch.tensor([(4, 4)]).to(z)

    scale = 4.
    centers = [(1, 0), (-1, 0), (0, 1), (0, -1), (1. / math.sqrt(2), 1. / math.sqrt(2)),
               (1. / math.sqrt(2), -1. / math.sqrt(2)), (-1. / math.sqrt(2),
                                                         1. / math.sqrt(2)), (-1. / math.sqrt(2), -1. / math.sqrt(2))]
    centers = [(scale * x, scale * y) for x, y in centers]
    centers = torch.tensor(centers).to(z)

    # centers = torch.tensor([(4, 4)]).to(z)

    # batch_size = z.shape[0]
    # z_logprob = []
    # normalizing_factor = torch.tensor(1./(2.*math.pi))
    # for i in range(batch_size):
    #     a = torch.sum((z[i] - centers).pow(2), dim=1)/2.
    #     z_logprob.append(torch.log(torch.mean(torch.exp(-a)*normalizing_factor).unsqueeze(dim=0)))
    #
    # z_logprob = torch.cat(z_logprob, dim=0)

    z = z.unsqueeze(dim=1).expand(-1, centers.shape[0], -1) - centers
    a = torch.sum(z.pow(2), dim=2) / 2.
    a = torch.exp(-a)
    a = torch.log(torch.mean(a, dim=1))

    z_logprob = a - torch.log(torch.tensor(2.*math.pi))
    # print(torch.norm(z_logprob2 - z_logprob)/torch.norm(z_logprob))

    return z_logprob

def gaussian_logprob(z, mu=DEFAULT_MU, sigma=DEFAULT_SIGMA):
    mu = mu.to(z)
    sigma = sigma.to(z)

    assert mu.shape[0] == sigma.shape[0]

    #TODO: implement this!
    assert(False)
    mu = torch.tensor([(4., 4.)]).to(z)
    sigma_half = torch.tensor([(2., 1.), (1., 2.)]).to(z)
    sigma_half_inv = torch.inverse(sigma_half).to(z)
    sigma = sigma_half.matmul(sigma_half).to(z)
    sigma_inv = torch.inverse(sigma).to(z)
    sigma_det = torch.det(sigma)

    logZ = -0.5 * math.log(2 * math.pi)
    return logZ - sigma_inv.matmul(z-mu) / 2.

ext: py

reason: import numpy


text:
import numpy as np
import json
from copy import deepcopy
from team import Team

result_type = ['Wk','0', '1', '2', '3', '4', '6']
result_prob = [6,   35,  39,  18,  4,   13,   5]
result_prob = result_prob/np.sum(result_prob) # sum up to 1

matchid = 0

class Match:

    def __init__(self, teama=None, teamb=None):
        self.complete = False
        self.winner = None
        self.batsman_score = {}
        self.bowler_score = {}
        self.partnerships = {}

        global matchid
        self.id = matchid
        matchid += 1

        self.init(teama, teamb)

    def init( self, teama, teamb ):
        if teama==None or teamb==None:
            return

        self.team1 = deepcopy( teama )
        self.team2 = deepcopy( teamb )

        coin = np.random.randint(0,2)
        if coin==1:
            self.team1, self.team2 = self.team2, self.team1

        self.toss = self.team2

    def __str__( self ):
        s = '#'+str(self.id)+' '
        if self.complete:
            if self.by_runs:
                s+=self.winner.name+ ' beat '+ self.looser.name+ ' by '+ str(self.by_runs)+' runs!'
            else:
                s+=self.winner.name+ ' beat '+ self.looser.name+ ' by '+ str(self.by_wickets)+' wickets!'
        else:
            s+=self.team1.name+ ' vs '+self.team2.name
        return s

    def __repr__( self ):
        return self.__str()

    def record( self, over, ball, runs, batsman, doubling, bowler, out=False):
        assert batsman in self.batsman_score, batsman
        assert bowler in self.bowler_score, bowler

        self.batsman_score[batsman]['runs'].append( runs )
        self.bowler_score[bowler]['runs'].append( runs )

        part = min(batsman, doubling)+','+max(batsman, doubling)
        self.partnerships[part].append( runs )

        if out:
            self.batsman_score[batsman]['out'] = True
            self.batsman_score[batsman]['bowler'] = bowler
            self.bowler_score[bowler]['wickets'].append( batsman )
        else:
            self.bowler_score[bowler]['wickets'].append( '' )

    def get_batsman_score( self, batsman ):
        if batsman not in self.batsman_score:
            return -1, -1, None

        bw = self.batsman_score[batsman]['bowler']
        if bw:
            bw = 'b '+ bw
        else:
            bw = '*\t'
        return sum(self.batsman_score[batsman]['runs']), len(self.batsman_score[batsman]['runs']), bw

    def print_scoreboard(self):
        print( 'Match between', self.team1.name, 'and', self.team2.name )
        for team in [self.team1, self.team2]:
            print(team.name+'\'s inings...')
            for i in range(1, 12):
                p = team.get_batsman(i)
                bsr, bsb, bsbw = self.get_batsman_score( p )
                if bsr<0:
                    continue
                print( '   '+str(i)+'. '+p+'\t'+bsbw+'\t'+str(bsr)+' ('+str(bsb)+')' )

            print()

            for bowler, score in self.bowler_score.items():
                if score['team']==team:
                    continue
                bls = len(score['runs'])
                bo = bls//6
                bb = bls%6
                runs = sum(score['runs'])
                wkts = np.count_nonzero(score['wickets'])

                print( '   '+bowler+'\t'+str(bo)+'.'+str(bb)+' - '+str(runs)+' - '+str(wkts) )

            print()

            print('  '+team.name, 'scored', str(team.score)+'/'+str(team.wickets), 'in '+str(team.overs)+'.'+str(team.balls))

            print()

        maxs=[]
        maxp=None
        for k,v in self.partnerships.items():
            if sum(v)>sum(maxs):
                maxs = v
                maxp = k
        print( '  Best Partnership: '+ maxp+'\t'+ str(sum(maxs))+' ('+str(len(maxs))+')' )

        print('  Man of the Match: '+self.mom[0]+'\t'+self.mom[1]+' ('+self.mom[2]+')')

        print ('  '+self.__str__())

        print()

    def summary( self ):
        assert self.complete, self

        s = '#'+str(self.id)+' '+self.winner.name+' ('+str(self.winner.score)+'/'+str(self.winner.wickets)+') beat '+ self.looser.name+' ('+str(self.looser.score)+'/'+str(self.looser.wickets)+')  by '
        if self.by_runs:
            s += str(self.by_runs)+' runs!\n'
        else:
            s += str(self.by_wickets)+' wickets!\n'

        s += '  Man of the Match: '+self.mom[0]+'\t'+self.mom[1]+' ('+self.mom[2]+')'

        return s

    def play( self, verbose=False ):
        self.team1.score = self.team2.score = None

        for teambt, teambl in [[self.team1, self.team2],[self.team2, self.team1]]:
            teambt.score = 0
            teambt.wickets = 0

            batsman_facing = teambt.get_batsman(1)
            batsman_doubling = teambt.get_batsman(2)
            next_batsman_index = 3

            self.batsman_score[batsman_facing]={'team':teambt, 'runs':[],'out':False, 'bowler':None}
            self.batsman_score[batsman_doubling]={'team':teambt, 'runs':[],'out':False, 'bowler':None}

            part = min(batsman_facing, batsman_doubling)+','+max(batsman_facing, batsman_doubling)
            self.partnerships[part]=[]

            for over in range(20):

                bowler = teambl.get_bowler(over+1)
                if bowler not in self.bowler_score:
                    self.bowler_score[bowler] = {'team':teambl, 'runs':[], 'wickets':[]}

                for ball in range(1,7):
                    r = np.random.choice(result_type, p=result_prob)

                    if r=='Wk':
                        teambt.wickets += 1

                        self.record( over, ball, 0, batsman_facing, batsman_doubling, bowler, True )

                        if teambt.wickets > 9:
                            break

                        batsman_facing = teambt.get_batsman(next_batsman_index)
                        self.batsman_score[batsman_facing] = {'team':teambt, 'runs':[],'out':False, 'bowler':None}

                        # new partnership
                        part = min(batsman_facing, batsman_doubling)+','+max(batsman_facing, batsman_doubling)
                        self.partnerships[part]=[]

                        next_batsman_index += 1

                    else:
                        r = int(r)
                        # add runs scored
                        teambt.score += r

                        self.record( over, ball, r, batsman_facing, batsman_doubling, bowler )

                        if teambl.score and teambt.score>teambl.score:
                            break

                        # swap batsmen if odd runs
                        if r==1 or r==3:
                            batsman_facing, batsman_doubling = batsman_doubling, batsman_facing

                if teambt.wickets > 9:
                    break

                if teambl.score and teambt.score>teambl.score:
                    break

                # swap the batmen at the end of the over
                batsman_facing, batsman_doubling = batsman_doubling, batsman_facing

            teambt.overs = over if ball<6 else over+1
            teambt.balls = ball if ball<6 else 0

        # result
        self.complete = True
        if self.team1.score>self.team2.score:
            self.winner = self.team1
            self.looser = self.team2
            self.by_runs = self.team1.score-self.team2.score
            self.by_wickets = None
        # elif self.score[1] > self.score[0]:
        # TODO: Handle ties
        else:
            self.winner = self.team2
            self.looser = self.team1
            self.by_runs = None
            self.by_wickets = 10-self.team2.wickets

        # man of the match
        mm, mms, mmb = self.get_motm()
        self.mom = (mm, str(mms), str(mmb))

        self.check_global_records()

        if verbose:
            self.print_scoreboard()

        return self.winner

    def check_global_records(self):
        pass

    def get_motm(self):
        maxn = None
        maxs = 0
        maxb = 0
        for n, b in self.batsman_score.items():
            r, bl, _ = self.get_batsman_score( n )
            if r>maxs:
                maxn = n
                maxs = r
                maxb = bl
            elif r==maxs and bl<maxb:
                maxn = n
                maxs = r
                maxb = bl

        return maxn, maxs, maxb

if __name__ == '__main__':
    with open('data.txt') as f:
        data = json.load(f)
        teams = []
        for team in data['Teams']:
            teams.append( Team(team, data=data) )

    m = Match( teams[0], teams[2] )
    m.play()
    print(m.summary())

max_stars_repo_name: aole/cricket

max_issues_repo_name: aole/cricket

max_forks_repo_name: aole/cricket


text:
import numpy as np
import matplotlib.pyplot as plt

img_array = np.load('car.npy')
rimg = np.reshape(img_array[0], (28, 28))

plt.imshow(rimg, cmap="gray")
plt.show()

print(type(rimg))
print(rimg)


text:
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

import torch

from ncc.data.summarization.language_pair_dataset import LanguagePairDataset

# from .translation import load_langpair_dataset, TranslationTask
from ncc.tasks.summarization.summarization import SummarizationTask
from ncc.tasks import register_task
from ncc.data import indexed_dataset
from ncc.data.bpe_dictionary import (BPE_Dictionary, SENTENCEPIECE)
from ncc.data.tools import data_utils
from ncc.data.wrappers.append_token_dataset import AppendTokenDataset
from ncc.data.wrappers.truncate_dataset import TruncateDataset
from ncc.data.wrappers.portion_dataset import PortionDataset
from ncc.data.wrappers.prepend_token_dataset import PrependTokenDataset
from ncc.data.summarization.language_pair_dataset import LanguagePairDataset
from ncc.tokenizers import tokenization
from functools import lru_cache
import torch
from ncc.data.ncc_dataset import NccDataset
from ncc import LOGGER
import numpy as np
import os

EVAL_BLEU_ORDER = 4

class IndexedRawTextDataset(NccDataset):
    """Takes a text file as input and binarizes it in memory at instantiation.
    Original lines are also kept in memory"""

    def __init__(self, path, dictionary, append_eos=True, reverse_order=False):
        self.tokens_list = []
        self.lines = []
        self.sizes = []
        self.append_eos = append_eos
        self.reverse_order = reverse_order
        self.read_data(path, dictionary)
        self.size = len(self.tokens_list)

    def read_data(self, path, dictionary):
        with open(path, 'r', encoding='utf-8') as f:
            for line in f:
                self.lines.append(line.strip('\n'))
                tokens = dictionary.encode_line(
                    line, tokenization._space_tokenizer, add_if_not_exist=False,
                    append_eos=self.append_eos, reverse_order=self.reverse_order,
                ).long()
                self.tokens_list.append(tokens)
                self.sizes.append(len(tokens))
        self.sizes = np.array(self.sizes)

    def check_index(self, i):
        if i < 0 or i >= self.size:
            raise IndexError('index out of range')

    @lru_cache(maxsize=8)
    def __getitem__(self, i):
        self.check_index(i)
        return self.tokens_list[i]

    def get_original_text(self, i):
        self.check_index(i)
        return self.lines[i]

    def __del__(self):
        pass

    def __len__(self):
        return self.size

    def num_tokens(self, index):
        return self.sizes[index]

    def size(self, index):
        return self.sizes[index]

    @staticmethod
    def exists(path):
        return os.path.exists(path)

def _load_dataset(path, impl, dict):
    if impl == 'raw':
        src_dataset = IndexedRawTextDataset(path=path, dictionary=dict)
    elif impl == 'mmap':
        # mmap dataset has been numberized, no need for dict
        src_dataset = indexed_dataset.MMapIndexedDataset(path=path)
    else:
        raise NotImplementedError("No such {} dataset implementation.".format(impl))
    return src_dataset

def load_langpair_dataset(
    data_path, split,
    src, src_dict,
    tgt, tgt_dict,
    dataset_impl,
    # combine, dataset_impl, upsample_primary,
    left_pad_source, left_pad_target,
    max_source_positions, max_target_positions,
    prepend_bos=False, load_alignments=False,
    truncate_source=False, append_source_id=False,
    truncate_target=False,
    append_eos_to_target=False,
    portion=None,
):
    # load source dataset
    src_path = os.path.join(data_path, '{}.{}'.format(split, src))
    src_dataset = _load_dataset(path=src_path, impl=dataset_impl, dict=src_dict)

    if truncate_source:
        LOGGER.info('truncate {}.{} to {}'.format(split, src, max_source_positions))
        src_dataset = TruncateDataset(src_dataset, max_source_positions)

    if portion is not None and split == 'train':
        LOGGER.info('set {}.{} portion to {}'.format(split, src, portion))
        src_dataset = PortionDataset(src_dataset, portion)

    # load target dataset
    tgt_path = os.path.join(data_path, '{}.{}'.format(split, tgt))
    tgt_dataset = _load_dataset(path=tgt_path, impl=dataset_impl, dict=tgt_dict)
    if truncate_target:
        LOGGER.info('truncate {}.{} to {}'.format(split, tgt, max_target_positions))
        tgt_dataset = TruncateDataset(tgt_dataset, max_target_positions)

    if prepend_bos:
        assert hasattr(src_dict, "bos_index") and hasattr(tgt_dict, "bos_index")
        src_dataset = PrependTokenDataset(src_dataset, src_dict.bos())
        if tgt_dataset is not None:
            tgt_dataset = PrependTokenDataset(tgt_dataset, tgt_dict.bos())

    eos = None
    if append_source_id:
        src_dataset = AppendTokenDataset(src_dataset, src_dict.index('[{}]'.format(src)))
        if tgt_dataset is not None:
            tgt_dataset = AppendTokenDataset(tgt_dataset, tgt_dict.index('[{}]'.format(tgt)))
        eos = tgt_dict.index('[{}]'.format(tgt))

    if portion is not None and split == 'train':
        LOGGER.info('set {}.{} portion to {}'.format(split, tgt, portion))
        tgt_dataset = PortionDataset(tgt_dataset, portion)

    # align_dataset = None
    # if load_alignments:
    #     align_path = os.path.join(data_path, '{}.align.{}-{}'.format(split, src, tgt))
    #     if indexed_dataset.dataset_exists(align_path, impl=dataset_impl):
    #         align_dataset = data_utils.load_indexed_dataset(align_path, None, dataset_impl)

    tgt_dataset_sizes = tgt_dataset.sizes if tgt_dataset is not None else None

    LOGGER.info('loaded {} examples from: {}'.format(len(src_dataset), src_path))
    LOGGER.info('loaded {} examples from: {}'.format(len(tgt_dataset), tgt_path))
    return LanguagePairDataset(
        src_dataset, src_dataset.sizes, src_dict,
        tgt_dataset, tgt_dataset_sizes, tgt_dict,
        left_pad_source=left_pad_source,
        left_pad_target=left_pad_target,
        max_source_positions=max_source_positions,
        max_target_positions=max_target_positions,
        align_dataset=None, eos=eos,
        remove_eos_from_source=True,
        append_eos_to_target=append_eos_to_target,
        shuffle=True,
        # shuffle=False,  # debug
    )

@register_task('summarization_from_pretrained_bart')
class SummarizationFromPretrainedBARTTask(SummarizationTask):
    """
    Translate from source language to target language with a model initialized with a multilingual pretrain.

    Args:
        src_dict (~fairseq.data.Dictionary): dictionary for the source language
        tgt_dict (~fairseq.data.Dictionary): dictionary for the target language

    .. note::

        The translation task is compatible with :mod:`fairseq-train`,
        :mod:`fairseq-generate` and :mod:`fairseq-interactive`.

    The translation task provides the following additional command-line
    arguments:

    .. argparse::
        :ref: fairseq.tasks.translation_parser
        :prog:
    """

    @staticmethod
    def add_args(parser):
        """Add task-specific arguments to the parser."""
        # fmt: off
        SummarizationTask.add_args(parser)
        parser.add_argument('--langs', required=True, metavar='LANG',
                            help='comma-separated list of monolingual language, for example, "en,de,fr"'
                                 'be careful these langs are what you used for pretraining (the same order),'
                                 'not for finetuning.'
                                 'you should always add all pretraining language idx during finetuning.')
        # fmt: on

    def __init__(self, args, src_dict, tgt_dict):
        super().__init__(args, src_dict, tgt_dict)
        self.langs = args.langs.split(',')
        for d in [src_dict, tgt_dict]:
            for l in self.langs:
                d.add_symbol('[{}]'.format(l))
            d.add_symbol('<mask>')

    def load_dataset(self, split, epoch=1, combine=False, **kwargs):
        """Load a given dataset split.

        Args:
            split (str): name of the split (e.g., train, valid, test)
        """
        paths = self.args.data.split(':')
        assert len(paths) > 0
        data_path = paths[(epoch - 1) % len(paths)]

        # infer langcode
        src, tgt = self.args.source_lang, self.args.target_lang

        self.datasets[split] = load_langpair_dataset(
            data_path, split, src, self.src_dict, tgt, self.tgt_dict,
            dataset_impl=self.args['dataset']['dataset_impl'],
            left_pad_source=self.args['task']['left_pad_source'],
            left_pad_target=self.args['task']['left_pad_target'],
            max_source_positions=self.args['task']['max_source_positions'],
            max_target_positions=self.args['task']['max_target_positions'],
            load_alignments=self.args['task']['load_alignments'],
            truncate_source=self.args['task']['truncate_source'],
            truncate_target=self.args['task']['truncate_target'],
            append_eos_to_target=self.args['task']['append_eos_to_target'],
            portion=self.args['dataset'].get('portion', None),
        )

    # def build_generator(self, args):
    #     if getattr(args, 'score_reference', False):
    #         from fairseq.sequence_scorer import SequenceScorer
    #         return SequenceScorer(
    #             self.target_dictionary,
    #             eos=self.tgt_dict.index('[{}]'.format(self.target_lang))
    #         )
    #     else:
    #         from fairseq.sequence_generator import SequenceGenerator
    #         return SequenceGenerator(
    #             self.target_dictionary,
    #             beam_size=getattr(args, 'beam', 5),
    #             max_len_a=getattr(args, 'max_len_a', 0),
    #             max_len_b=getattr(args, 'max_len_b', 200),
    #             min_len=getattr(args, 'min_len', 1),
    #             normalize_scores=(not getattr(args, 'unnormalized', False)),
    #             len_penalty=getattr(args, 'lenpen', 1),
    #             unk_penalty=getattr(args, 'unkpen', 0),
    #             temperature=getattr(args, 'temperature', 1.),
    #             match_source_len=getattr(args, 'match_source_len', False),
    #             no_repeat_ngram_size=getattr(args, 'no_repeat_ngram_size', 0),
    #             eos=self.tgt_dict.index('[{}]'.format(self.args.target_lang))
    #         )

    def build_dataset_for_inference(self, src_tokens, src_lengths):
        src_lang_id = self.source_dictionary.index('[{}]'.format(self.args.source_lang))
        source_tokens = []
        for s_t in src_tokens:
            s_t = torch.cat([s_t, s_t.new(1).fill_(src_lang_id)])
            source_tokens.append(s_t)
        dataset = LanguagePairDataset(src_tokens, src_lengths, self.source_dictionary)
        return dataset

max_stars_repo_name: code-backdoor/code-backdoor


text:
# Copyright 2013 Devsim LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# The purpose is to verify our triangle element field calculation.
# It is based on Laux's weighting scheme
#@article{Laux:1985,
# author = {Laux, Steven E. and Byrnes, Robert G.},
# title = {Semiconductor device simulation using generalized mobility models},
# journal = {IBM J. Res. Dev.},
# issue_date = {May 1985},
# volume = {29},
# number = {dim},
# month = may,
# year = {1985},
# issn = {0018-8646},
# pages = {289--301},
# numpages = {13},
# url = {http://dx.doi.org/10.1147/rd.293.0289},
# doi = {10.1147/rd.293.0289},
# acmid = {1012099},
# publisher = {IBM Corp.},
# address = {Riverton, NJ, USA},
#}

import sys
try:
  import numpy
  import numpy.linalg
except:
  print("numpy is not available with your installation and is not being run")
  sys.exit(-1)

from devsim import *
from laux_common import *
import laux_common

load_devices(file="gmsh_diode3d_dd.msh")
device = "diode3d"
region = "Bulk"

SetDimension(3)
nee = laux_common.nee
nen = laux_common.nen
dim = laux_common.dim

# There are nee edges per tetrahedron
#number_elements = len(scalar_efield)/laux_common.nee;

number_test = -1

RunTest(device, region, number_test)

#row = 0
#if True:
##for row in range(10):
#  col = 4
#  sl1 = slice(nee*row,nee*(row+1))
#  sl2 = slice(dim*col,dim*(col+1))
#  print output[(sl1, sl2)]
#  print output_compare[(sl1,sl2)]
#  print output[(sl1, sl2)] - output_compare[(sl1,sl2)]
#

#print "VERIFY node_index@n0 == node_index@en0"
## creates node_index@n0, node_index@n1
#element_model(device=device, region=region, name="scalar_edge_index", equation="edge_index")
#edge_from_node_model(node_model="node_index", device=device, region=region)
#scalar_edge_index = get_element_model_values(device=device, region=region, name="scalar_edge_index")
#scalar_edge_index = [int(x) for x in scalar_edge_index]
#scalar_node_index0 = get_edge_model_values(device=device, region=region, name="node_index@n0")
#scalar_node_index0 = [int(x) for x in scalar_node_index0]
#scalar_node_index1 = get_edge_model_values(device=device, region=region, name="node_index@n1")
#scalar_node_index1 = [int(x) for x in scalar_node_index1]
#eedge = scalar_edge_index[sl1]
#print eedge
##print scalar_edge_index[sl1]
#print [scalar_node_index0[x] for x in eedge]
#print [scalar_node_index1[x] for x in eedge]
#print node_indexes[0][sl1]
#print node_indexes[1][sl1]

max_stars_repo_name: QuantumOfMoose/devsim

max_issues_repo_name: QuantumOfMoose/devsim

max_forks_repo_name: QuantumOfMoose/devsim


text:
import numpy as np
import matplotlib.pyplot as plt
import time as tm

def logistic_func(r,x):
  return r*x*(1-x)

def normal(mu,sigma, x):
    norm_ = 1/np.sqrt(2*np.pi*sigma*sigma)
    return norm_*np.exp(-(x-mu)**2/(2*sigma*sigma))

def xnext(r,xo,N):
  no = 0
  X = np.zeros(r.size*N)
  A = np.zeros(r.size*N)
#Iterate for each value of r in the array of total_of_r (7000) values.
  for r_value in r:
    xinit = xo
#For each specific r value, iterate 200 x to get a starting value of X and A.
    for i in range(200):
      xinit = normal(0,r_value,xinit)
    X[no] = xinit
    A[no] = r_value
    no += 1
#Now, for that r-value, iterate it N-times through the function.
#This gives N-values for each r-value.
    for j in range(1,N):
      X[no] = normal(0,r_value,X[no-1])
      A[no]= r_value
      no += 1
  return A, X
#Fixed Values
#=========================
xo = 1.4
Num = 400
r_min = -1
r_max = 1
total_of_r = 20000
r = np.linspace(r_min,r_max,total_of_r)
#+++++++++++++++++++++++++

A, X = xnext(r,xo,Num)
#Plot of results
#+++++++++++++++++++++++++
fig, ax = plt.subplots(figsize=(8,8))
ax.set_facecolor(plt.cm.gray(.9))
ax.plot(A,X,",",markersize=0.1,color="green")
ax.set_xlim(r_min,r_max)
ax.set_ylim(-1,10)
ax.set_title(r'Bifurcation plot of Normal function: $x_{n + 1} = N(\mu, \sigma, \ x_n)$')
ax.set_xlabel(r'$\sigma$', size = 14)
ax.set_ylabel(r'$f( \mu, \sigma, \ x_n )$',size=14)
plt.show()
#plt.savefig('Bifur_plot.png')

max_stars_repo_name: yuchiaol/Non-linear-dynamics-Strogatz


text:
from __future__ import division, print_function

import os
import torch
import pickle as pkl
from tqdm import tqdm
from PIL import Image
import numpy as np
from pdb import set_trace

from util import util
from util import obj_io

def main_test_with_gt_smpl(in_dir, out_dir, pretrained_checkpoint, pretrained_gcmr_checkpoint):
    from evaluator import Evaluator
    from dataloader.dataloader_cape import TestingImgLoader

    os.makedirs(out_dir, exist_ok=True)

    device = torch.device("cuda")
    loader = TestingImgLoader(in_dir)
    evaluator = Evaluator(device, pretrained_checkpoint, pretrained_gcmr_checkpoint)
    for step, batch in enumerate(tqdm(loader, desc='Testing', total=len(loader), initial=0)):
        batch = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in batch.items()}
        mesh = evaluator.test_cape_pifu(batch['image'], 256, batch['verts'], batch['tedra'], batch['calib'])
        mesh_fname = os.path.join(out_dir, batch['dataset'][0], f"{batch['subject'][0]}_{batch['rotation'][0]:03d}.obj")
        os.makedirs(os.path.dirname(mesh_fname), exist_ok=True)
        obj_io.save_obj_data(mesh, mesh_fname)

        smpl_param = {}
        for key in batch.keys():
            if 'smpl_' in key:
                smpl_param[key] = batch[key][0]

        np.save(mesh_fname[:-4], smpl_param, allow_pickle=True)

    print('Testing Done. ')

def main_test_wo_gt_smpl_with_optm(test_img_dir, out_dir, pretrained_checkpoint, pretrained_gcmr_checkpoint,
                                   iternum=50):
    from evaluator import Evaluator
    from dataloader.dataloader_testing import TestingImgLoader

    smpl_vertex_code, smpl_face_code, smpl_faces, smpl_tetras = \
        util.read_smpl_constants('./data')

    os.makedirs(out_dir, exist_ok=True)

    device = torch.device("cuda")
    loader = TestingImgLoader(test_img_dir, 512, 512, white_bg=True)
    evaluator = Evaluator(device, pretrained_checkpoint, pretrained_gcmr_checkpoint)

    # for iter in np.arange(1,100,10):
    iter = 100
    for step, batch in enumerate(tqdm(loader, desc='Testing', total=len(loader), initial=0)):
        batch = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in batch.items()}
        pred_betas, pred_rotmat, scale, trans, pred_smpl = evaluator.test_gcmr(batch['img'])
        optm_thetas, optm_betas, optm_smpl = evaluator.optm_smpl_param(
            batch['img'], batch['keypoints'], pred_betas, pred_rotmat, scale, trans, iter)
        optm_betas = optm_betas.detach()
        optm_thetas = optm_thetas.detach()
        scale, trans = scale.detach(), trans.detach()
        mesh = evaluator.test_pifu(batch['img'], 256, optm_betas, optm_thetas, scale, trans)

        img_dir = batch['img_dir'][0]
        img_fname = os.path.split(img_dir)[1]
        img = Image.fromarray((255.0*batch['img'][0]).permute(1,2,0).detach().cpu().numpy().astype(np.uint8))

        mesh_fname = os.path.join(out_dir, img_fname[:-4] + f"_{iter}.obj")
        init_smpl_fname = os.path.join(out_dir, img_fname[:-4] + f"_{iter}_init_smpl.obj")
        optm_smpl_fname = os.path.join(out_dir, img_fname[:-4] + f"_{iter}_optm_smpl.obj")
        img_preview_fname = os.path.join(out_dir, img_fname[:-4] + f"_{iter}.png")

        # export
        img.save(img_preview_fname)
        obj_io.save_obj_data(mesh, mesh_fname)
        obj_io.save_obj_data({'v': pred_smpl.squeeze().detach().cpu().numpy(), 'f': smpl_faces},
                            init_smpl_fname)
        obj_io.save_obj_data({'v': optm_smpl.squeeze().detach().cpu().numpy(), 'f': smpl_faces},
                            optm_smpl_fname)

        smpl_param_name = os.path.join(out_dir, img_fname[:-4] + f'_{iter}_smplparams.pkl')
        with open(smpl_param_name, 'wb') as fp:
            pkl.dump({'betas': optm_betas.squeeze().detach().cpu().numpy(),
                    'body_pose': optm_thetas.squeeze().detach().cpu().numpy(),
                    'init_betas': pred_betas.squeeze().detach().cpu().numpy(),
                    'init_body_pose': pred_rotmat.squeeze().detach().cpu().numpy(),
                    'body_scale': scale.squeeze().detach().cpu().numpy(),
                    'global_body_translation': trans.squeeze().detach().cpu().numpy()},
                    fp)
        # os.system('cp %s %s.original' % (mesh_fname, mesh_fname))
        # os.system('%s %s %s' % (REMESH_BIN, mesh_fname, mesh_fname))
        # os.system('%s %s %s' % (ISOLATION_REMOVAL_BIN, mesh_fname, mesh_fname))
    print('Testing Done. ')

def main_test_texture(test_img_dir, out_dir, pretrained_checkpoint_pamir,
                      pretrained_checkpoint_pamirtex):
    from evaluator_tex import EvaluatorTex
    from dataloader.dataloader_testing import TestingImgLoader

    os.makedirs(out_dir, exist_ok=True)

    device = torch.device("cuda")
    loader = TestingImgLoader(test_img_dir, 512, 512, white_bg=True)
    evaluater = EvaluatorTex(device, pretrained_checkpoint_pamir, pretrained_checkpoint_pamirtex)
    for step, batch in enumerate(tqdm(loader, desc='Testing', total=len(loader), initial=0)):
        batch = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in batch.items()}
        if not ('betas' in batch and 'pose' in batch):
            raise FileNotFoundError('Cannot found SMPL parameters! You need to run PaMIR-geometry first!')
        if not ('mesh_vert' in batch and 'mesh_face' in batch):
            raise FileNotFoundError('Cannot found the mesh for texturing! You need to run PaMIR-geometry first!')

        mesh_color = evaluater.test_tex_pifu(batch['img'], batch['mesh_vert'], batch['betas'],
                                             batch['pose'], batch['scale'], batch['trans'])

        img_dir = batch['img_dir'][0]
        img_fname = os.path.split(img_dir)[1]
        mesh_fname = os.path.join(out_dir, img_fname[:-4] + '_tex.obj')
        obj_io.save_obj_data({'v': batch['mesh_vert'][0].squeeze().detach().cpu().numpy(),
                              'f': batch['mesh_face'][0].squeeze().detach().cpu().numpy(),
                              'vc': mesh_color.squeeze()},
                             mesh_fname)
    print('Testing Done. ')

if __name__ == '__main__':

    import argparse

    # loading cfg file
    parser = argparse.ArgumentParser()

    parser.add_argument(
        '-indir', '--input_dir', type=str, help='path of test dir')
    parser.add_argument(
        '-outdir', '--output_dir', type=str, default="../results/", help='path of test dir')

    args = parser.parse_args()

    iternum=50
    input_dir = args.input_dir
    output_dir = args.output_dir

    #! NOTE: We recommend using this when accurate SMPL estimation is available (e.g., through external optimization / annotation)
    main_test_with_gt_smpl(input_dir,
                           output_dir,
                           pretrained_checkpoint='./results/pamir_geometry/checkpoints/latest.pt',
                           pretrained_gcmr_checkpoint='./results/gcmr_pretrained/gcmr_2020_12_10-21_03_12.pt')

    # #! Otherwise, use this function to predict and optimize a SMPL model for the input image
    # main_test_wo_gt_smpl_with_optm(input_image_dir,
    #                                output_dir,
    #                                pretrained_checkpoint='./results/pamir_geometry/checkpoints/latest.pt',
    #                                pretrained_gcmr_checkpoint='./results/gcmr_pretrained/gcmr_2020_12_10-21_03_12.pt',
    #                                iternum=iternum)

    # main_test_texture(output_dir,
    #                   output_dir,
    #                   pretrained_checkpoint_pamir='./results/pamir_geometry/checkpoints/latest.pt',
    #                   pretrained_checkpoint_pamirtex='./results/pamir_texture/checkpoints/latest.pt')

max_stars_repo_name: YuliangXiu/PaMIR

max_issues_repo_name: YuliangXiu/PaMIR

max_forks_repo_name: YuliangXiu/PaMIR


text:
import logging
import sys
import threading
import time
from collections import deque
import pandas as pd
from datetime import datetime
# ---
from win10toast import ToastNotifier

import pandas as pd
import numpy as np
import pickle
import os
import asyncio
import datetime
from datetime import datetime
from datetime import timedelta, timezone
from typing import Optional
from dotenv import load_dotenv
from operator import itemgetter

load_dotenv()
PWD = os.getenv("PWD")
db_name = PWD + "\\database" + "\\RVNUSDT.db"
import sys

sys.path.insert(1, PWD + "\\modules")
from alg_modules.alg_handler import AlgHandler
from plot_modules.candle_plot import CandlePlot
from collections import deque
from paper_trade import PaperTrader
import time
import logging

DEBUG = __debug__
LOG_FILE_NAME = "log_file_name.log"
format = "%(asctime)s [%(levelname)s]: %(message)s"
logger = logging.basicConfig(
    filename=LOG_FILE_NAME if not DEBUG else None,
    format=format,
    encoding="utf-8",
    level=logging.INFO,
)
if not DEBUG:
    logging.getLogger(logger).addHandler(logging.StreamHandler())

from stop_loss import StopLoss
from trade_strategy import TradeStrategy
from wss_thread import WssThread
from api_modules.open_binance_api import OpenBinanceApi
import pytz
tzdata = pytz.timezone('Europe/Moscow')

# ---
class Trader(object):
    '''docstring for Trader'''
    def __init__(self, ):
        super().__init__()

        self.is_stopped = None
        self._thread = threading.Thread(target=self.between_callback, args=())
        # self._thread = threading.Thread(target=asyncio.run, args=())
        self._lock = threading.Lock()

    def between_callback(self):
        self.loop = asyncio.new_event_loop()
        asyncio.set_event_loop(self.loop)

        self.loop.run_until_complete(self.thread_function(self))
        self.loop.close()

    async def thread_function(self, *args, **kwargs):
        # ====

        def compute_timedelta(dt: datetime):
            if dt.tzinfo is None:
                dt = dt.astimezone()
            now = datetime.now(timezone.utc)
            return max((dt - now).total_seconds(), 0)

        async def sleep_until(when: datetime, result = None):
            """|coro|
            Sleep until a specified time.
            If the time supplied is in the past this function will yield instantly.
            .. versionadded:: 1.3
            Parameters
            -----------
            when: :class:`datetime.datetime`
                The timestamp in which to sleep until. If the datetime is naive then
                it is assumed to be local time.
            result: Any
                If provided is returned to the caller when the coroutine completes.
            """
            delta = compute_timedelta(when)
            return await asyncio.sleep(delta, result)

        # ====
        server_time = datetime.fromtimestamp(OpenBinanceApi.server_time()/1000)
        local_time = datetime.now()
        delay = server_time - local_time
        # ====
        # notifications
        toast = ToastNotifier()
        static_notification_settings = dict(
            title="Algo traid BOT",
            duration = 20,
            icon_path = "python.ico",
            threaded = 1,
        )

        notify = lambda msg: toast.show_toast(
            msg=msg,
            **static_notification_settings,
            )

        msg="Watch out for notifications from here"
        async def notification(msg):
            if not notify(msg):
                await asyncio.sleep(20)
                notify(msg)

        await notification(msg)
        # ====
        DATA_AWAIT_TIME = 1 # seconds
        SERVER_DELAY = 10 # seconds
        INTERVAL_SECONDS = 60 # seconds
        # request that data from api
        w = WssThread(
            url='wss://stream.binance.com:9443/ws/rvnusdt@ticker',
            maxlen=10,
            )
        w.start()

        STOP_LOSS_ENABLED=True
        STOP_LOSS_THRESHOLD=-1.3

        DEQUE_MAX_LENGTH = 200
        INTERVAL = '1m'

        df = OpenBinanceApi.get_df(
            pair = 'RVNUSDT',
            interval = INTERVAL,
            limit = 1000,
            )
        # drop last row TODO make assert to not dublicate last row from cycle
        df = df[:-1]
        stop_loss_trade_flag = False
        MA_list = (2, 7, 25, 100)

        window = deque(maxlen=200)
        for i, row in df.iterrows():
            window.append(dict(row.squeeze()))
        #initial currency resources
        p_trdr = PaperTrader(
            main_currency_label='RVN',
            secondary_currency_label='USD',
            main_currency_amount=100,
            secondary_currency_amount=0,
            fee=0.1,
        )
        trade_data = pd.DataFrame(
            columns = p_trdr.get_df(timestamp=df.iloc[-1]['Date']).columns.values
        )
        stop_loss = StopLoss(
            STOP_LOSS_THRESHOLD=STOP_LOSS_THRESHOLD,
        )
        # init alg
        alg = AlgHandler(
            df=pd.DataFrame([]),
            MA_list=MA_list,
            )
        while not self._stopped:
                logging.info('===get new data===')
                new_df = OpenBinanceApi.get_df(
                        pair = 'RVNUSDT',
                        interval = INTERVAL,
                        limit = 2,
                    )
                dt = datetime.fromtimestamp(int(new_df.Real_Date[-1:])/1000)
                server_time = datetime.fromtimestamp(OpenBinanceApi.server_time()/1000)
                logging.debug(f'server time: {server_time}   {server_time.minute=}, {dt.minute=}')
                # extract function?

                if server_time.minute == dt.minute:
                    logging.debug('+++===success===+++')
                    window.append(dict(new_df[-2:-1].squeeze()))
                    df_ = pd.DataFrame(window)
                    # === process data here ===
                    # display(df_)
                    for _, row in df_.iterrows():
                        # window.append(dict(row))
                        # df__ = pd.DataFrame(window)
                        alg.update_data(df_)
                        alg.calculate(val_col='Open', time_col='Date',)
                        do_trade, cross_type = alg.evaluate()
                        if STOP_LOSS_ENABLED:
                            stop_loss_trade_flag, trade_data = stop_loss.stop_loss_alg(
                                                                    trade_data=trade_data,
                                                                    p_trdr=p_trdr,
                                                                    row=row,
                                                                )

                    if do_trade:
                        trade_data, stop_loss_trade_flag = TradeStrategy.trade_alg(stop_loss_trade_flag, trade_data, p_trdr, row, cross_type)
                        await notification(f'Trade done: {trade_data}\n {trade_data[-1:]}')
                    logging.info(f'do trade: {do_trade}')
                    self.p_trdr = p_trdr
                    self.alg = alg
                    # self.df__ = df__
                    self.df_ = df_
                    self.window = window
                    # display(trade_data)
                    # display(alg.crosses)
                    # === end of data processing ===
                    time_to_sleep = dt - delay + timedelta(seconds=SERVER_DELAY) + timedelta(seconds=INTERVAL_SECONDS)
                    server_delay = dt - server_time
                    logging.debug(f'server valid time: {server_time}')
                    logging.debug(f'server delay: {server_delay.total_seconds()}')
                    logging.debug(f'sleep till: {time_to_sleep}')
                    await sleep_until(time_to_sleep)
                else:
                    logging.debug('---not valid---')
                    logging.debug('sleep 1 sec')
                    await asyncio.sleep(DATA_AWAIT_TIME)

    def get_data(self, ) -> dict:
        with self._lock:
            # e = (self.queue or [None])[-1]
            try:
                # return(ast.literal_eval(e)) #???
                return self.p_trdr, self.alg, self.df_, self.window
            except ValueError as err:
                logging.debug(err)

    def start(self):
        '''Start the thread'''
        logging.info(f"[{self.__class__.__name__}] Opening thread")
        self.is_stopped = False
        self._thread.start()

    def close(self) -> None:
        ''' Close the thread'''
        with self._lock:
            logging.info(f"[{self.__class__.__name__}] Closing thread")
            self.is_stopped = True
            # self.loop.keep_running = False
            loop = asyncio.get_event_loop()
            loop.stop()
            loop.close()

    @property
    def _stopped(self):
        return self.is_stopped

    @property
    def is_alive(self):
        return self._thread.is_alive()

if __name__ == "__main__":
    format = "%(asctime)s: %(message)s"
    logging.basicConfig(format=format, level=logging.INFO,
                        datefmt="%H:%M:%S")
    w = Trader(
        # url='wss://stream.binance.com:9443/ws/rvnusdt@ticker',
        # maxlen=10,
        )
    w.start()

    try:
        while True:
            print(w.get_data())
            time.sleep(5)
    finally:
        # close thread
        w.close()


text:
"""[4.2.3] High-dimensional nonlinear elliptic PDEs
Implemented with Pytorch. (torch version >= 1.8.1)

* Variable interpretation:

- x: torch.Tensor, (Number of points, dimension)
-
"""
import sys, os
from copy import deepcopy
import random
from random import randint
import json
from tqdm import tqdm
import numpy as np

import torch
import torch.nn as nn
import torch.optim as optim
from torch.autograd import grad, Variable
import matplotlib.pyplot as plt

from train import train

# >>> global params definition >>>
PARAMS = {
    'name': 'EllipticDirichlet',
    'dim': 20,
    'left boundary': -1,
    'right boundary': 1,
    'K_primal': 1,
    'K_adv': 1,
    'lr_primal': 0.0015,
    'lr_adv': 0.004,
    'Nr': None,
    'Nb': None,
    'alpha': None,
    'use elu': False,
    'n_iter': 3000,
}
# update
PARAMS['Nr'] = PARAMS['dim'] * 4000
PARAMS['Nb'] = PARAMS['dim'] * PARAMS['dim'] * 40

PARAMS['alpha'] = PARAMS['Nb'] * 20000

DEVICE = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
# <<< global params definition <<<

torch.set_default_tensor_type(torch.DoubleTensor)
random.seed(0)
torch.manual_seed(0)
torch.cuda.manual_seed(0)
np.random.seed(0)

# >>> Numerical function definition >>>
def a(x: torch.Tensor):
    """
    :1+|x|^2 by definition
    Sum along the dimension axis
    """
    a0 = x**2
    return 1 + torch.sum(a0, dim=1)

def p0(x: torch.Tensor):
    """
    :\rho_0^2 by definition
    Only the first two dimensions contributes
    """
    return np.pi * x[:, 0]**2 + x[:, 1]**2  / 2.0

def p1(x: torch.Tensor):
    """
    :\rho_1^2 by definition
    Only the first two dimensions contributes
    """
    return ((np.pi)**2 * x[:, 0]**2 + x[:, 1]**2)  / 4.0

def f(x):
    """
    R.H.S of PDE
    """
    _p0 = p0(x)
    _p1 = p1(x)
    _a = a(x)
    _cos = torch.cos(_p0)
    return 4.0 * _p1 * _a * torch.sin(_p0) - \
            (4.0 * _p0 + (np.pi + 1) * _a) * _cos + \
                2.0 * _p1 * _cos**2

def g(x):
    """
    :u* Dirichlet boundary / ground truth function
    """
    return torch.sin( (np.pi * x[:, 0]**2 + x[:, 1]**2) / 2.0 )

def loss_all(xr: torch.Tensor, xb: torch.Tensor, u_theta, phi_eta, alpha, device):
    """
    Args:
        torch.Tensor: (Nr x d)
        torch.Tensor: (Nb x d)
        Network instance
        Network instance
        alpha: weight constant
    Returns:
        torch.Tensor: (1 x 1)
    """
    # Calculate derivative w.r.t. to x
    xr = Variable(xr, requires_grad=True)

    # Calculate derivative w.r.t. to x[x1, x2, ...]
    _out_u_theta = torch.sum(u_theta(xr).squeeze())
    _grad_u_theta = grad(_out_u_theta, xr, create_graph=True)[0]

    _out_phi_eta = torch.sum(phi_eta(xr).squeeze())
    _grad_phi_eta = grad(_out_phi_eta, xr, create_graph=True)[0]

    # feed forward
    _phi_eta = phi_eta(xr).squeeze()              # comp. graph => loss
    _u_theta_bdry = u_theta(xb).squeeze()        # comp. graph => loss

    # >>> PDE-specific: calculate for I (integrand) >>>
    #t1 = torch.sum(_grad_u_theta * _grad_phi_eta , dim=1) * a(xr)  # norm, sum along dimension
    t1_list = []
    for i in range(xr.shape[1]):
        for j in range(xr.shape[1]):
            t1_list.append(_grad_u_theta[:, i] * _grad_phi_eta[:, j] * a(xr))

    t2 = torch.sum(_grad_u_theta * _grad_u_theta, dim=1) / 2.0
    I = (t2 - f(xr)) * _phi_eta - sum(t1_list)
    # <<< PDE-specific: calculate for I (integrand) <<<

    loss_int = 2 * (torch.log(I.norm()) - torch.log(_phi_eta.norm()))
    loss_bdry = (_u_theta_bdry - g(xb)).norm()**2  / xb.shape[0]

    return loss_int + loss_bdry * alpha
# <<< Numerical function definition <<<

if __name__ == '__main__':
    train(params=PARAMS,
            g=g,
            loss_func=loss_all,
            device=DEVICE)


text:
import engformat as ef
import matplotlib.pyplot as plt
import numpy as np
import eqsig
from sfsidb import deprecated_load as load
from bwplot import cbox, lsbox
import bwplot

# THIS FILE HAS DEPRECATED

def add_ylabel(fname):
    if "PPT" in fname:
        return "Pore pressure [Pa]"
    elif "ACC" in fname:
        return "Acceleration [m/s2]"
    return fname

def basic(sub_plot, folder_path, fname, dbset, label=None, ccbox=None, lls=None, quiet=False):
    if quiet:
        series, time = load.load_record(folder_path + fname, dbset, quiet=True)
        if series is None:  # Could not find file
            return
    else:
        series, time = load.load_record(folder_path + fname, dbset)
    if series is None:
        raise FileNotFoundError
    if label is None:
        label = fname

    kwargs = bwplot.pack_plotting_kwargs(ccbox, lls)

    sub_plot.plot(time, series, label=label, **kwargs)
    ef.time_series(sub_plot)
    ef.revamp_legend(sub_plot)

    plt.xlabel('Time [s]')
    plt.ylabel(add_ylabel(fname))

def fourier_spectrum(sub_plot, folder_path, fname, dbset, label=None, ccbox=None, lls=None, quiet=False):
    if quiet:
        series0, time = load.load_record(folder_path + fname, dbset, quiet=True)
        if series0 is None:  # Could not find file
            return
    else:
        series0, time = load.load_record(folder_path + fname, dbset)

    dt = time[1] - time[0]
    if label is None:
        label = fname

    kwargs = bwplot.pack_plotting_kwargs(ccbox, lls)

    acc0 = eqsig.AccSignal(series0, dt)
    sub_plot.loglog(acc0.fa_frequencies, np.abs(acc0.fa_spectrum), label=label, **kwargs)
    sub_plot.loglog(acc0.smooth_fa_frequencies, acc0.smooth_fa_spectrum, label=label + "-smooth", **kwargs)

def transfer_function(sub_plot, folder_path, fname0, fname1, dbset, label=None, significant=True, quiet=False,
                      ccbox=False, lls=False):
    if quiet:
        series0, time = load.load_record(folder_path + fname0, dbset, quiet=True)
        series1, time = load.load_record(folder_path + fname1, dbset, quiet=True)
        if series0 is None or series1 is None:
            print("Could not find file")
            return
    else:
        series0, time = load.load_record(folder_path + fname0, dbset)
        series1, time = load.load_record(folder_path + fname1, dbset)
    dt = time[1] - time[0]
    if label is None:
        label = fname1

    acc0 = eqsig.AccSignal(series0, dt)
    fas0_smooth = np.abs(acc0.smooth_fa_spectrum)
    ffs_smooth = acc0.smooth_fa_frequencies

    acc1 = eqsig.AccSignal(series1, dt)
    fas1_smooth = np.abs(acc1.smooth_fa_spectrum)

    h_smooth = fas1_smooth / fas0_smooth

    kwargs = bwplot.pack_plotting_kwargs(ccbox, lls)

    if significant:
        # Only plot range of frequencies that are high
        min_freq_i, max_freq_i = eqsig.significant_range(fas1_smooth)
        sub_plot.semilogx(ffs_smooth[min_freq_i:max_freq_i], h_smooth[min_freq_i:max_freq_i], label=label, **kwargs)
    else:
        sub_plot.semilogx(ffs_smooth, h_smooth, label=label, **kwargs)
    sub_plot.set_ylabel("H: " + fname0)

def pore_pressure_series(sub_plot, folder_path, fname, dbset, label=None, ccbox=None, lls=None, quiet=False):
    if quiet:
        series, time = load.load_record(folder_path + fname, dbset, quiet=True)
        if series is None:  # Could not find file
            return
    else:
        series, time = load.load_record(folder_path + fname, dbset)
    if "PPT" not in fname:
        raise ValueError("PPT record required for pore_pressure view")

    if label is None:
        label = fname

    kwargs = bwplot.pack_plotting_kwargs(ccbox, lls)

    sub_plot.plot(time, series, label=label, **kwargs)
    ef.time_series(sub_plot)
    ef.revamp_legend(sub_plot)

    plt.xlabel('Time [s]')
    plt.ylabel("Pore pressure [Pa]")

def total_stress(sub_plot, folder_path, esigy_fname, ppt_fname, dbset, label=None, ccbox=None, lls=None, quiet=False):
    if quiet:
        esigy_series, time = load.load_record(folder_path + esigy_fname, dbset, quiet=True)
        ppt_series, time = load.load_record(folder_path + ppt_fname, dbset, quiet=True)
        if esigy_series is None or ppt_series is None:  # Could not find file
            return
    else:
        esigy_series, time = load.load_record(folder_path + esigy_fname, dbset, quiet=False)
        ppt_series, time = load.load_record(folder_path + ppt_fname, dbset, quiet=False)
    if label is None:
        label = ppt_fname

    kwargs = bwplot.pack_plotting_kwargs(ccbox, lls)
    del kwargs['ls']
    del kwargs['c']

    sub_plot.fill_between(time, -esigy_series + ppt_series, 0, label="Total stress", **kwargs)
    # sub_plot.plot(time, -esigy_series + ppt_series, 0, label="Total stress", **kwargs)
    sub_plot.fill_between(time, -esigy_series, label="Effective stress", **kwargs)
    # sub_plot.plot(time, ppt_series, label="Pore-pressure", **kwargs)
    ef.time_series(sub_plot)
    ef.revamp_legend(sub_plot)

    plt.xlabel('Time [s]')
    #plt.ylabel(add_ylabel(fname))

def ru_series(sub_plot, folder_path, fname, dbset, label=None, ccbox=None, lls=None, quiet=False):
    if quiet:
        series, time = load.load_record(folder_path + fname, dbset, quiet=True)
        if series is None:  # Could not find file
            return
    else:
        series, time = load.load_record(folder_path + fname, dbset)
    if "ESIG" not in fname:
        raise ValueError("ESIG record required for ru view")

    esigy = series[0]  # initial effective vertical stress
    # calculating ru (ru=1-(esigy/esigy_initial)
    ru_vals = 1 - (series / esigy)

    kwargs = bwplot.pack_plotting_kwargs(ccbox, lls)
    # plotting time vs ru
    sub_plot.plot(time, ru_vals, label=label, **kwargs)

    sub_plot.xlabel('Time [s]')
    sub_plot.ylabel('Ru')

def gamma_v_tau(sub_plot, folder_path, strs_fname, tau_fname, dbset, label=None, ccbox=None, lls=None, quiet=False):
    if quiet:
        strs, time = load.load_record(folder_path + strs_fname, dbset, quiet=True)
        tau, time = load.load_record(folder_path + tau_fname, dbset, quiet=True)
        if strs is None or tau is None:
            print("Could not find file")
            return
    else:
        strs, time = load.load_record(folder_path + strs_fname, dbset)
        tau, time = load.load_record(folder_path + tau_fname, dbset)

    if label is None:
        label = tau_fname
    kwargs = bwplot.pack_plotting_kwargs(ccbox, lls)
    sub_plot.plot(strs, tau, label=label, **kwargs)
    sub_plot.set_xlabel('Shear strain [%]')
    sub_plot.set_ylabel('Shear stress [kPa]')

max_stars_repo_name: eng-tools/sfsidb

max_issues_repo_name: eng-tools/sfsidb

max_forks_repo_name: eng-tools/sfsidb


text:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import tensorflow as tf
import numpy as np
import tensorflow.contrib.slim as slim
from gym import spaces
from Networks import P_Network
from Networks import Q_Network
from ray.rllib.models.model import Model
from ray.rllib.utils.annotations import override
from ray.rllib.models.catalog import ModelCatalog
from ray.rllib.models.preprocessors import Preprocessor
from ray.rllib.agents.ddpg.ddpg_policy_graph import DDPGPolicyGraph
from ray.rllib.models.misc import normc_initializer

class Q_KOMP(DDPGPolicyGraph):

    @override(DDPGPolicyGraph)
    def _build_q_network(self, inputs, observation_space, action_space, actions):
        XYZlim = 0.8   # limits for states, don't use for goals
        X_max = 1.1    # limit for both goals in x
        Ylim = 0.50    # limit for both goals in y
        Zlim = 1.0     # upper limit for both goals in z
        NZlim = -0.1   # lower limit for both goals in z
        observation_space = spaces.Dict({'achieved': spaces.Box(low=np.tile(np.array([-0.1, -Ylim, NZlim, -0.1, -0.1]),(5,1)),
                                                                       high=np.tile(np.array([X_max, Ylim, Zlim, 1.1, 1.1]),(5,1)), dtype=np.float32),
                                               'desired': spaces.Box(low=np.array([-0.1, -Ylim, NZlim, -0.1, -0.1]),
                                                                       high=np.array([X_max, Ylim, Zlim, 1.1, 1.1]), dtype=np.float32),
                                               'image': spaces.Box(low=0, high=1.1, shape=(234, 234, 4), dtype=np.float32),
                                               'states': spaces.Box(low=np.array([-XYZlim, -XYZlim, 0, -1.1, -0.1]),
                                                                    high=np.array([XYZlim, XYZlim, XYZlim, 1.1, 1.1]), dtype=np.float32),
                                            })
        q_net = Q_Network({'obs':inputs,'prev_actions':actions,}, observation_space, action_space, 1, {"free_log_std":False})
        return q_net.outputs, q_net

    @override(DDPGPolicyGraph)
    def _build_p_network(self, inputs, obs_space, action_space):

        XYZlim = 0.8   # limits for states, don't use for goals
        X_max = 1.1    # limit for both goals in x
        Ylim = 0.50    # limit for both goals in y
        Zlim = 1.0     # upper limit for both goals in z
        NZlim = -0.1   # lower limit for both goals in z
        observation_space = spaces.Dict({'achieved': spaces.Box(low=np.tile(np.array([-0.1, -Ylim, NZlim, -0.1, -0.1]),(5,1)),
                                                                       high=np.tile(np.array([X_max, Ylim, Zlim, 1.1, 1.1]),(5,1)), dtype=np.float32),
                                               'desired': spaces.Box(low=np.array([-0.1, -Ylim, NZlim, -0.1, -0.1]),
                                                                       high=np.array([X_max, Ylim, Zlim, 1.1, 1.1]), dtype=np.float32),
                                               'image': spaces.Box(low=0, high=1.1, shape=(234, 234, 4), dtype=np.float32),
                                               'states': spaces.Box(low=np.array([-XYZlim, -XYZlim, 0, -1.1, -0.1]),
                                                                    high=np.array([XYZlim, XYZlim, XYZlim, 1.1, 1.1]), dtype=np.float32),
                                            })

        p_net = P_Network({'obs':inputs,}, observation_space, action_space, 5, {"free_log_std":False})
        return p_net.outputs, p_net


text:
import os, fnmatch, sys, time

import dill as pickle

import scipy.interpolate as interp

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.mlab as mlab

import bead_util as bu
import calib_util as cu
import configuration as config

import time

dirname = '/data/20180612/new_sync2'

elec_ind = 3
pos_ind = 0  # {0: x, 1: y, 2: z}

files = bu.find_all_fnames(dirname)
files = bu.sort_files_by_timestamp(files)

#files = ['/data/20180611/bead4/1_4mbar_xyzcool.h5']

for filname in files:
    df = bu.DataFile()
    df.load(filname, plot_sync=True)

    freqs = np.fft.rfftfreq(df.nsamp, d=1.0/df.fsamp)
    fft_x = np.fft.rfft(df.pos_data[0])
    fft_elec = np.fft.rfft(df.electrode_data[3])
    plt.loglog(freqs, np.abs(fft_x))
    plt.loglog(freqs, np.abs(fft_elec))
    plt.grid(alpha=0.6)

    plt.show()

reason: import numpy,import scipy


text:
from enum import IntEnum
from typing import NamedTuple
import itertools
import math
from scipy.special import erfinv

from quantlib.algorithms.ana.ops import NoiseType, ForwardComputationStrategy
from manager.doeflows.experimentaldesign import ExperimentalSetup, ExperimentalDesign, patch_dictionary

from typing import List, Tuple

# ====================
#  DEGREES OF FREEDOM
# ====================

class Period(IntEnum):
    # SHORT    = 20
    STANDARD = 50
    # LONG     = 80

class NoiseMean(IntEnum):
    STATIC  = 0
    DYNAMIC = 1

class NoiseVariance(IntEnum):
    STATIC  = 0  # gradients flow in the same way throughout the training
    DYNAMIC = 1

class DecayIntervals(IntEnum):
    OVERLAPPED = 0
    SAME_START = 1
    SAME_END   = 2
    PARTITION  = 3

class DecayPowerLaw(IntEnum):
    HOMOGENEOUS = 0
    PROGRESSIVE = 1  # theory-inspired

NoisePolicy = NamedTuple('NoisePolicy', [('mi',    NoiseMean),
                                         ('sigma', NoiseVariance)])

DecayPolicy = NamedTuple('DecayPolicy', [('di',    DecayIntervals),
                                         ('dpl',   DecayPowerLaw)])

Policy      = NamedTuple('Policy',      [('fcs',   ForwardComputationStrategy),
                                         ('np',    NoisePolicy),
                                         ('dp',    DecayPolicy)])

def get_noise_hyperparameters(n_layers: int,
                              layer_id: int,
                              nt:       NoiseType,
                              np:       NoisePolicy,
                              dp:       DecayPolicy):

    # compute decay powers
    if dp.dpl == DecayPowerLaw.HOMOGENEOUS:
        base_power = 1
    elif dp.dpl == DecayPowerLaw.PROGRESSIVE:
        assert 0 <= layer_id < n_layers
        base_power = n_layers - layer_id
    else:
        raise ValueError

    mi_alpha    = 1 * base_power
    sigma_alpha = 2 * base_power

    # I compare different mass distributions in a given interval
    half_interval           = 0.5   # probability mass should be contained in the interval [mi_beta - half_interval, mi_beta + half_interval]
    mass_fraction_unbounded = 0.95  # probability distributions with unbounded support can not be "squeezed" inside the interval, so I try to put a "reasonable" amount of mass in it

    # compute mean hyper-parameters
    if np.mi == NoiseMean.STATIC:
        mi_beta  = 0.0
        mi_alpha = 0
    elif np.mi == NoiseMean.DYNAMIC:
        mi_beta = -half_interval  # anneal to the Heaviside H+
    else:
        raise ValueError

    # compute standard deviation (or standard deviation proxy) hyper-parameters
    if nt == NoiseType.UNIFORM:
        # for the uniform noise, I don't use the standard deviation, but a proxy s = \sqrt{3} * std
        sigma_beta = half_interval  # STE-like
    elif nt == NoiseType.TRIANGULAR:
        # for the triangular noise, I don't use the standard deviation, but a proxy s = \sqrt{6} * std
        sigma_beta = half_interval
    elif nt == NoiseType.LOGISTIC:
        sigma_beta = 1 / (2 * math.log((1 + mass_fraction_unbounded) / (1 - mass_fraction_unbounded)))
    elif nt == NoiseType.NORMAL:
        sigma_beta = - half_interval * math.sqrt(2) / (2 * erfinv(-mass_fraction_unbounded))
    else:
        raise ValueError
    assert sigma_beta > 0.0  # if there is no noise, gradients can't flow!

    if np.sigma == NoiseVariance.STATIC:
        sigma_alpha = 0
    elif np.sigma == NoiseVariance.DYNAMIC:
        pass
    else:
        raise ValueError

    return (mi_beta, mi_alpha), (sigma_beta, sigma_alpha)

def compute_noise_hyperparameters(n_layers: int,
                                  nt:       NoiseType,
                                  np:       NoisePolicy,
                                  dp:       DecayPolicy) -> List[Tuple[Tuple[float, int], Tuple[float, int]]]:
    return [get_noise_hyperparameters(n_layers, layer_id, nt, np, dp) for layer_id in range(0, n_layers)]

def compute_decay_intervals(n_layers: int,
                            period:   int,
                            dp:       DecayPolicy) -> List[Tuple[int, int]]:

    if dp.di == DecayIntervals.OVERLAPPED:
        starts = [period * 0        for layer_id in range(0, n_layers)]
        ends   = [period * n_layers for layer_id in range(0, n_layers)]

    elif dp.di == DecayIntervals.PARTITION:
        starts = [period * layer_id       for layer_id in range(0, n_layers)]
        ends   = [period * (layer_id + 1) for layer_id in range(0, n_layers)]

    elif dp.di == DecayIntervals.SAME_START:
        starts = [period * 0              for layer_id in range(0, n_layers)]
        ends   = [period * (layer_id + 1) for layer_id in range(0, n_layers)]

    elif dp.di == DecayIntervals.SAME_END:
        starts = [period * (n_layers - (layer_id + 1)) for layer_id in range(0, n_layers)]
        ends   = [period * n_layers                    for layer_id in range(0, n_layers)]

    else:
        raise ValueError

    decay_intervals = [(s, e) for s, e in zip(starts, ends)]

    return decay_intervals

def compute_timer_specs(layers: List[List[str]],
                        period: int,
                        nt:     NoiseType,
                        policy: Policy):

    noise_hyperparameters = compute_noise_hyperparameters(len(layers), nt, policy.np, policy.dp)
    decay_intervals       = compute_decay_intervals(len(layers), period, policy.dp)

    timer_specs = []
    for layer_id, (modules, ((mb, ma), (sb, sa)), (s, e)) in enumerate(zip(layers, noise_hyperparameters, decay_intervals)):
        timer_spec = dict()
        timer_spec['modules'] = modules
        timer_spec['mi'] = \
        {
            'beta': mb,
            'fun': 'bws',
            'kwargs': {
                   'tstart': s,
                   'tend':   e,
                   'alpha':  ma
            }
        }
        timer_spec['sigma'] = \
        {
            'beta': sb,
            'fun': 'bws',
            'kwargs': {
                'tstart': s,
                'tend':   e,
                'alpha':  sa
            }
        }
        timer_specs.append(timer_spec)

    return timer_specs

class ANACIFAR10VGG8(ExperimentalDesign):

    def __init__(self):
        dofs = (Period, NoiseType, ForwardComputationStrategy, NoiseMean, NoiseVariance, DecayIntervals, DecayPowerLaw)
        super(ANACIFAR10VGG8, self).__init__(dofs)

    def _generate_experimental_setups(self):

        setups = list()

        layers = [
            # conv/linear     activation
            ['pilot.0',      'pilot.2'],
            ['features.1',   'features.3'],
            ['features.4',   'features.6'],
            ['features.8',   'features.10'],
            ['features.11',  'features.13'],
            ['classifier.0', 'classifier.2'],
            ['classifier.3', 'classifier.5']
        ]

        n_layers     = len(layers)
        magic_number = 10 - n_layers

        noise_policies = {NoisePolicy(nm, nv)  for nm, nv      in itertools.product(NoiseMean, NoiseVariance)}
        decay_policies = {DecayPolicy(di, dpl) for di, dpl     in itertools.product(DecayIntervals, DecayPowerLaw)}
        policies       = {Policy(fcs, np, dp)  for fcs, np, dp in itertools.product(ForwardComputationStrategy, noise_policies, decay_policies)}

        dynamic_policies = set(filter(lambda p: not (p.np.mi == NoiseMean.STATIC and p.np.sigma == NoiseVariance.STATIC), policies))
        static_policies  = policies.difference(dynamic_policies)
        dynamic_policies = set(filter(lambda p: not (p.np.sigma == NoiseVariance.STATIC and p.fcs == ForwardComputationStrategy.EXPECTATION), dynamic_policies))
        static_policies  = set(filter(lambda p: p.fcs != ForwardComputationStrategy.EXPECTATION and p.dp.di == DecayIntervals.OVERLAPPED and p.dp.dpl == DecayPowerLaw.HOMOGENEOUS, static_policies))

        for period, nt in itertools.product(Period, NoiseType):

            n_epochs = period * (n_layers + magic_number)
            training_n_epochs_patch = \
                {
                    'training': {
                        'n_epochs': n_epochs
                    }
                }

            hand_scheduler_epoch = period * (n_layers + 1)  # lower the learning rate to fine-tune the parameters
            training_gd_patch = \
                {
                    'training': {
                        'gd': {
                            'lr_sched': {
                                'class': 'HandScheduler',
                                'kwargs': {'schedule': {hand_scheduler_epoch: 0.1}}
                            }
                        }
                    }
                }

            # generate setups and patches
            for policy in (static_policies | dynamic_policies):

                network_quantize_patch = \
                    {
                        'network': {
                            'quantize': {
                                'kwargs': {
                                    'noise_type': nt.name.lower(),
                                    'strategy':   policy.fcs.name.lower()
                                }
                            }
                        }
                    }

                timer_specs = compute_timer_specs(layers, period, nt, policy)
                training_quantize_patch = \
                    {
                        'training': {
                            'quantize': {
                                'kwargs': {'ctrl_spec': timer_specs}
                            }
                        }
                    }

                setup = (period.name, nt.name, policy.fcs.name, policy.np.mi.name, policy.np.sigma.name, policy.dp.di.name, policy.dp.dpl.name)
                patch = {**network_quantize_patch}
                patch = patch_dictionary(patch, training_n_epochs_patch)
                patch = patch_dictionary(patch, training_gd_patch)
                patch = patch_dictionary(patch, training_quantize_patch)

                setups.append(ExperimentalSetup(dofs_values=setup, config_patch=patch))

        return setups

max_stars_repo_name: mdatres/quantlab

max_issues_repo_name: mdatres/quantlab

max_forks_repo_name: mdatres/quantlab

reason: from scipy


text:
import os.path
import math
import argparse
import time
import random
import numpy as np
from collections import OrderedDict
import logging
from torch.utils.data import DataLoader
from utils import util_option as option
import torch
import cv2
from models.network_swinir import SwinFaceRestoration as net
from data.test_dataset import test_dataset
from PIL import Image
def tensor2uint(img):
    img = img.data.squeeze().float().clamp_(0, 1).cpu().numpy()
    if img.ndim == 3:
        img = np.transpose(img, (1, 2, 0))
    return np.uint8((img*255.0).round())

def main():

    parser = argparse.ArgumentParser()
    parser.add_argument('--task', type=str, default='noise', help='classical_sr, lightweight_sr, real_sr, '
                                                                     'gray_dn, color_dn, jpeg_car')

    parser.add_argument('--model_path', type=str,
                        default='check_points_new/noise/models/300000_G.pth')

    args = parser.parse_args()

    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

    model = define_model(args)
    model.eval()
    model = model.to(device)

    # setup folder and path
    save_dir = 'results'+'/'+args.task

    window_size = 8
    os.makedirs(save_dir, exist_ok=True)

    # for i in range(1,7):
    #     os.mkdir('results'+'/'+args.task+'/'+str(i))
    test_set = test_dataset()
    test_loader = DataLoader(test_set,
                                batch_size=1,
                                shuffle=False,
                                num_workers=2,
                                drop_last=True,
                                pin_memory=True)

    for i, test_data in enumerate(test_loader):
        print(i)
        imgL = test_data['L'].to(device)
        mask = test_data['mask'].to(device)
        imgH = test_data['H'].to(device)

        with torch.no_grad():
            output = test(torch.cat((imgL,mask),dim=1), model)
            #output = test(imgL, model)
        out_dict = OrderedDict()
        out_dict['L'] = imgL.detach()[0].float().cpu()
        out_dict['E'] = output.detach()[0].float().cpu()
        out_dict['H'] = imgH.detach()[0].float().cpu()

        E_img = tensor2uint(out_dict['E'])
        H_img = tensor2uint(out_dict['H'])
        L_img = tensor2uint(out_dict['L'])
        out = np.concatenate((L_img,E_img,H_img), axis = 1)
        out = Image.fromarray(out)
        img_path = save_dir + '/' + test_data['name'][0]

        #cv2.imwrite(img_path, out)
        out.save(img_path)

def define_model(args):

    model = net(in_chans=3,
                img_size=512,
                window_size=8,
                embed_dim=48,
                num_heads=[1,2,4,8],
                mlp_ratio=2)
    param_key_g = 'params'

    pretrained_model = torch.load(args.model_path)
    model.load_state_dict(pretrained_model[param_key_g] if param_key_g in pretrained_model.keys() else pretrained_model, strict=True)

    return model

def test(img_lq, model):

    output = model(img_lq)

    return output

if __name__ == '__main__':
    main()


text:
# -*- coding: utf-8 -*-

#calculate pixel-based global accuracy
import numpy as np

def calGAcc(s_eta_true,s_eta_pre):
    s_eta_true = 1*(s_eta_true >=0.5)   #Binarization
    s_eta_pre = 1*(s_eta_pre >=0.5)   #Binarization

    n_sample = len(s_eta_true)
    nAcc = np.zeros(n_sample,)

    for i in range(0,n_sample):  #Calculate global accuracy for each instance.
        nAcc[i] = np.mean(1*np.equal(s_eta_true[i,:,:],s_eta_pre[i,:,:]))

    meanAcc = np.mean(nAcc)
    return meanAcc

max_stars_repo_path: Porosity in selective laser sintering/calGAcc.py


text:
#!/usr/bin/env python3
# -*- coding:utf-8 -*-
# =========================================================================== #
# Project : ML Studio                                                         #
# Version : 0.1.14                                                            #
# File    : test_activations.py                                               #
# Python  : 3.8.3                                                             #
# --------------------------------------------------------------------------  #
# Author  : John James                                                        #
# Company : DecisionScients                                                   #
# Email   : jjames@decisionscients.com                                        #
# URL     : https://github.com/decisionscients/MLStudio                       #
# --------------------------------------------------------------------------  #
# Created       : Monday, June 15th 2020, 10:24:08 pm                         #
# Last Modified : Monday, June 15th 2020, 10:24:25 pm                         #
# Modified By   : John James (jjames@decisionscients.com)                     #
# --------------------------------------------------------------------------  #
# License : BSD                                                               #
# Copyright (c) 2020 DecisionScients                                          #
# =========================================================================== #
"""Test Activation Functions."""
import math
import numpy as np
import pytest
from pytest import mark

from scipy.special import expit
from sklearn.utils.extmath import softmax

from mlstudio.supervised.algorithms.optimization.services.activations import Sigmoid, Softmax
# --------------------------------------------------------------------------  #
@mark.activations
@mark.sigmoid
class SigmoidActivationTests:

    def test_sigmoid_activation(self):
        # Get an array
        x = np.random.default_rng().uniform(low=0, high=20, size=20)
        # Expected result
        exp_result = expit(x)
        # Actual result
        act = Sigmoid()
        act_result = act(x)
        assert np.allclose(exp_result, act_result), "Sigmoid error"

@mark.activations
@mark.softmax
class SoftmaxActivationTests:

    def test_softmax_activation(self):
        # Get an array
        x = np.random.default_rng().uniform(low=0, high=20, size=(20,4))
        # Expected result
        exp_result = softmax(x)
        # Actual result
        act = Softmax()
        act_result = act(x)
        assert np.allclose(exp_result, act_result), "Softmax error"

max_issues_repo_name: DecisionScients/MLStudio

max_forks_repo_name: decisionscients/MLStudio

reason: import numpy,from scipy


text:
import numpy as np

_color_map_depths = np.array([
    [0, 0, 0],          # 0.000
    [0, 0, 255],        # 0.114
    [255, 0, 0],        # 0.299
    [255, 0, 255],      # 0.413
    [0, 255, 0],        # 0.587
    [0, 255, 255],      # 0.701
    [255, 255,  0],     # 0.886
    [255, 255,  255],   # 1.000
    [255, 255,  255],   # 1.000
]).astype(float)
_color_map_bincenters = np.array([
    0.0,
    0.114,
    0.299,
    0.413,
    0.587,
    0.701,
    0.886,
    1.000,
    2.000, # doesn't make a difference, just strictly higher than 1
])

def color_depth_map(depths, scale=None):
    """
    Color an input depth map.

    Arguments:
        depths -- HxW numpy array of depths
        [scale=None] -- scaling the values (defaults to the maximum depth)

    Returns:
        colored_depths -- HxWx3 numpy array visualizing the depths
    """

    if scale is None:
        scale = depths.max()

    values = np.clip(depths.flatten() / scale, 0, 1)
    # for each value, figure out where they fit in in the bincenters: what is the last bincenter smaller than this value?
    lower_bin = ((values.reshape(-1, 1) >= _color_map_bincenters.reshape(1,-1)) * np.arange(0,9)).max(axis=1)
    lower_bin_value = _color_map_bincenters[lower_bin]
    higher_bin_value = _color_map_bincenters[lower_bin + 1]
    alphas = (values - lower_bin_value) / (higher_bin_value - lower_bin_value)
    colors = _color_map_depths[lower_bin] * (1-alphas).reshape(-1,1) + _color_map_depths[lower_bin + 1] * alphas.reshape(-1,1)
    return colors.reshape(depths.shape[0], depths.shape[1], 3).astype(np.uint8)


text:
### Import modules
import numpy as np
import torch
import matplotlib.pyplot as plt

from pprint import pprint
from transformers import AutoTokenizer, AutoModelWithLMHead, pipeline
from .utils import utils
from .base import BaseClass

class Summarizer(BaseClass):

    def __init__(self, name='BART Summarizer',max_length=130,min_length=30):
        super().__init__(name)

        #Init name and metadata
        self.name = name
        self.max_length = max_length
        self.min_length = min_length

        #Create net
        self.tokenizer = AutoTokenizer.from_pretrained("facebook/bart-large-cnn")
        self.model = AutoModelWithLMHead.from_pretrained("facebook/bart-large-cnn")
        self.predictor = pipeline('summarization',
                                 model = self.model,
                                 tokenizer = self.tokenizer)

    def predict(self,text):
        """
        Does text summarization. In order to perform batch inference,
        you can either call this predict() function in a for-loop or alternatively (advanced)
        try to modify this predict() function to perform batch-inferencing.

        Input:
            text: str object. Seed text to be used for generation.

        Output:
            predictions: list object. Generated text.
        """

        #Infer
        output = self.predictor(text,max_length=self.max_length,min_length=self.min_length)

        return output

    def visualize(self,raw,output):
        """
        Simple function to call pretty-print for a neater text representation.

        Input:
            raw: str object; default text
            output: str object; obj returned by predict()

        Output:
            None
        """

        #Print!
        pprint({"Raw":raw,
                "Summarized":output[0]['summary_text']})


text:
"""
:Authors: - Iason
"""

from collections import defaultdict
import numpy as np

def inside(forest, topsort, omega=lambda edge: edge.log_prob):
    """
    Inside recursion.
    :param forest: an acyclic hypergraph.
    :param topsort: a partial ordering of the nodes in the forest.
    :param omega: a function that computes the weight of an edge (defaults to the edge's own log probability)
    :return: a dictionary mapping a symbol to its inside weight.
    """
    inside_prob = defaultdict(float)

    # visit nodes bottom up
    for parent in topsort:

        incoming = forest.get(parent, frozenset())

        # leaves have inside weight 1
        if not incoming:
            # log(1) = 0
            inside_prob[parent] = 0
        else:
            # log(0) = -inf
            total = -float("inf")

            for edge in incoming:
                w = sum((inside_prob[child] for child in edge.rhs), omega(edge))
                # log(a) + log(b) = log(exp(a) + exp(b))
                # total = log(exp(total) + exp(w))
                total = np.logaddexp(total, w)

            inside_prob[parent] = total

    return inside_prob


text:
import re

import fasttext
from flask import Flask, request, jsonify, render_template
import mwapi

app = Flask(__name__)
app.config["DEBUG"] = True
CUSTOM_UA = 'wikidata topic app -- isaac@wikimedia.org'
SESSION = mwapi.Session('https://www.wikidata.org',
                        user_agent=CUSTOM_UA)
FT_MODEL = fasttext.load_model('models/model.bin')

PROVIDE_EXPLANATIONS = False

print("Try: http://127.0.0.1:5000/api/v1/wikidata/topic?qid=Q72334&debug")

if PROVIDE_EXPLANATIONS:
    from lime.lime_text import LimeTextExplainer
    import numpy as np

    lbls_to_idx = {l.replace('__label__', ''):i for i,l in enumerate(FT_MODEL.get_labels())}
    EXPLAINER = LimeTextExplainer(class_names=[l.replace('__label__', '') for l in FT_MODEL.get_labels()])

@app.route('/')
def index():
    return 'Server Works!'

def adjust_topics_based_on_claims(topics, claims):
    joined_claims = [":".join(c) for c in claims]
    properties = [c[0] for c in claims]
    # list / disambiguation pages
    if 'P31:Q4167410' in joined_claims:
        topics = [('Compilation.List_Disambig', 1, 'P31:Q4167410 -- Disambiguation')] + topics
    elif 'P31:Q13406463' in joined_claims:
        topics = [('Compilation.List_Disambig', 1, 'P31:Q13406463 -- List')] + topics
    elif 'P360' in properties:
        topics = [('Compilation.List_Disambig', 1, 'P360 -- List')] + topics
    # geography only should apply to items with coordinates
    if 'P625' not in joined_claims:
        for idx in range(len(topics)):
            if topics[idx][0].startswith('Geography'):
                topics[idx] = (topics[idx][0], max(0, topics[idx][1] - 0.501), ' -- '.join([topics[idx][2], "downgraded bc no coords"]))
    # Culture.Biography.Women should not include men (at 0.5 threshold):
    if ('P21:Q6581097' in joined_claims or  # male
            'P21:Q2449503' in joined_claims or  # transgender male
            'P21:Q44148' in joined_claims or  # male organisms
            'P21:Q27679766' in joined_claims or  # transmasculine
            'P21:Q15145778' in joined_claims):  # cisgender male
        for idx in range(len(topics)):
            if topics[idx][0] == 'Culture.Biography.Women':
                topics[idx] = (topics[idx][0], min(0.49, topics[idx][1]), ' -- '.join([topics[idx][2], " downgraded bc male"]))
    topics = sorted(topics, key=lambda tup: tup[1], reverse=True)
    return topics, claims

@app.route('/api/v1/wikidata/topic', methods=['GET'])
def get_topics():
    qid, threshold, debug = validate_api_args()
    if validate_qid(qid):
        name, topics, claims = label_qid(qid, SESSION, FT_MODEL, threshold)
        topics, claims = adjust_topics_based_on_claims(topics, claims)
        if debug:
            return render_template('wikidata_topics.html',
                                   qid=qid, claims=claims, topics=topics, name=name)
        else:
            topics = [{'topic':t[0], 'score':t[1], 'explanation':t[2]} for t in topics]
            return jsonify(topics)
    return jsonify({'Error':qid})

def get_qid(title, lang, session=None):
    if session is None:
        session = mwapi.Session('https://{0}.wikipedia.org'.format(lang), user_agent=CUSTOM_UA)

    try:
        result = session.get(
            action="query",
            prop="pageprops",
            ppprop='wikibase_item',
            titles=title,
            format='json',
            formatversion=2
        )
    except Exception:
        print("Failed:", title)
        return "API call failed for {0}.wikipedia: {1}".format(lang, title)

    try:
        return result['query']['pages'][0]['pageprops'].get('wikibase_item', None)
    except (KeyError, IndexError):
        print("No results returned:", title)
        return "Title does not exist in {0}: {1}".format(lang, title)

def validate_qid(qid):
    return re.match('^Q[0-9]+$', qid)

def validate_api_args():
    if 'qid' in request.args:
        qid = request.args['qid'].upper()
        if not validate_qid(qid):
            qid = "Error: poorly formatted 'qid' field. {0} does not match 'Q#...'".format(qid)
    elif 'en_title' in request.args:
        qid = get_qid(request.args['en_title'], lang='en')
    else:
        qid = "Error: no 'qid' or 'en_title' field provided. Please specify."

    threshold = 0.5
    if 'threshold' in request.args:
        try:
            threshold = float(request.args['threshold'])
        except ValueError:
            threshold = "Error: threshold value provided not a float: {0}".format(request.args['threshold'])

    debug = False
    if 'debug' in request.args:
        debug = True
        threshold = 0

    return qid, threshold, debug

def predict_proba_lime(datapoints):
    lbl_to_idx = {l:i for i,l in enumerate(FT_MODEL.get_labels())}
    probabilities = np.zeros((len(datapoints), len(lbl_to_idx)))
    for i, dp in enumerate(datapoints):
        lbls, probs = FT_MODEL.predict(dp, k=-1)
        for lbl_idx, lbl in enumerate(lbls):
            probabilities[i][lbl_to_idx[lbl]] = probs[lbl_idx]
    return probabilities

def label_qid(qid, session, model, threshold=0.5, debug=False):
    # default results
    name = ""
    above_threshold = []
    claims_tuples = []

    # get claims for wikidata item
    result = {}
    try:
        result = session.get(
            action="wbgetentities",
            props='claims|labels',
            languages='en',
            languagefallback='',
            format='json',
            ids=qid
        )
    except Exception:
        print("Failed:", qid)
    if debug:
        print(result)

    if 'missing' in result['entities'][qid]:
        print("No results:", qid)
    else:
        # get best label
        for lbl in result['entities'][qid]['labels']:
            name = result['entities'][qid]['labels'][lbl]['value']
            print('{0}: {1}'.format(qid, name))
            break

        # convert claims to fastText bag-of-words format
        claims = result['entities'][qid]['claims']
        for prop in claims:  # each property, such as P31 instance-of
            included = False
            for statement in claims[prop]:  # each value under that property -- e.g., instance-of might have three different values
                try:
                    if statement['type'] == 'statement' and statement['mainsnak']['datatype'] == 'wikibase-item':
                        claims_tuples.append((prop, statement['mainsnak']['datavalue']['value']['id']))
                        included = True
                except Exception:
                    continue
            if not included:
                claims_tuples.append((prop, ))
        if not len(claims_tuples):
            claims_tuples = [('<NOCLAIM>', )]
        if debug:
            print(claims_tuples)
        claims_str = ' '.join([' '.join(c) for c in claims_tuples])

        # make prediction
        lbls, scores = model.predict(claims_str, k=-1)
        results = {l:s for l,s in zip(lbls, scores)}
        if debug:
            print(results)
        sorted_res = [(l.replace("__label__", ""), results[l], "None") for l in sorted(results, key=results.get, reverse=True)]
        above_threshold = [r for r in sorted_res if r[1] >= threshold]
        lbls_above_threshold = []
        if above_threshold:
            for res in above_threshold:
                print('{0}: {1:.3f} -- {2}'.format(*res))
                if res[1] > 0.5:
                    lbls_above_threshold.append(res[0])
        else:
            print("No label above {0} threshold.".format(threshold))
            print("Top result: {0} ({1:.3f}) -- {2}".format(sorted_res[0][0], sorted_res[0][1], sorted_res[0][2]))

        if PROVIDE_EXPLANATIONS and lbls_above_threshold:
            exp = EXPLAINER.explain_instance(claims_str, predict_proba_lime, num_features=5,
                                             labels=[lbls_to_idx[l] for l in lbls_above_threshold])
            for i,lbl in enumerate(lbls_above_threshold):
                above_threshold[i] = (above_threshold[i][0], above_threshold[i][1],
                                      '; '.join(['{0} ({1:.3f})'.format(ft[0], ft[1]) for ft in exp.as_list(label=lbls_to_idx[lbl])]))
                print(above_threshold[i])

    return name, above_threshold, claims_tuples

app.run()

max_stars_repo_name: geohci/wikidata-topic-model

max_issues_repo_name: geohci/wikidata-topic-model

max_forks_repo_name: geohci/wikidata-topic-model


text:
from numpy.testing import assert_array_equal
from pathlib import Path
from pydub import AudioSegment
from old.vl8.dsp import io
import numpy as np
import tdir
import unittest

TEST_FILE = Path(__file__).parent / 'b-4098.wav'
DIR = Path(__file__).parent / 'sources'

@tdir
class TestIO(unittest.TestCase):
    def test_file(self):
        length = 4098

        data, sample_rate = io.read(TEST_FILE)

        assert sample_rate == 44100
        assert data.shape == (2, length)
        segment = AudioSegment.from_file(TEST_FILE)
        assert segment._data == data.tobytes('F')

        filename = f'result-{length}.wav'
        io.write(filename, data)
        data2, _ = io.read(filename)

        assert_array_equal(data, data2)

    def test_mono(self):
        length = 7249
        data = np.linspace(0, length - 1, length, dtype=np.int16)
        data = data.reshape(1, length)
        assert data.shape == (1, length,)

        filename = f'result-{length}.wav'
        io.write(filename, data)
        data2, _ = io.read(filename)
        assert_array_equal(data, data2)

    def test_mono_stereo(self):
        length = 4
        mono = np.linspace(0, length - 1, length, dtype=np.int16)
        mono = mono.reshape(1, length)
        stereo = np.ones((2, length), dtype=np.int16)

        assert mono.shape == (1, length)

        stereo += mono
        expected = np.array([[1, 2, 3, 4], [1, 2, 3, 4]], dtype=np.int16)
        assert_array_equal(stereo.data, expected)

    def test_two(self):
        infile = DIR / '1.wav'
        outfile = Path('1.wav')
        segment = AudioSegment.from_file(infile)
        segment.export(outfile, format='wav')

        d1 = infile.read_bytes()
        d2 = outfile.read_bytes()
        assert len(d1) == len(d2)
        errors = [a for a in enumerate(zip(d1, d2)) if a[1][0] != a[1][1]]
        assert errors == [(4, (134, 133))]
        # AudioSegment changes one bit in the length from the sample

    def test_three(self):
        infile = DIR / '1.wav'
        outfile = Path('1.wav')
        data, sample_rate = io.read(infile)
        assert data.dtype == np.int8
        io.write(outfile, data, sample_rate)

        d1 = infile.read_bytes()
        d2 = outfile.read_bytes()
        assert len(d1) == len(d2)
        errors = [a for a in enumerate(zip(d1, d2)) if a[1][0] != a[1][1]]
        assert errors == [(4, (134, 133))]

    def NO_test_pydub_24(self):
        # A 24-bit file
        infile = DIR / 'sunnk - skinning gales 07.wav'
        outfile = Path('stereo.wav')
        segment = AudioSegment.from_file(infile)
        segment.export('stereo.wav', format='wav')
        assert infile.read_bytes() == outfile.read_bytes()

max_stars_repo_name: rec/sorta

max_issues_repo_name: rec/sorta

max_forks_repo_name: rec/sorta

reason: import numpy,from numpy