text:
"""Sequence to Sequence models."""
import glob
import numpy as np
import os

import torch
import torch.nn as nn
from torch.autograd import Variable

import src.decoders as decoders
import src.encoders as encoders

from src.cuda import CUDA

def get_latest_ckpt(ckpt_dir):
    ckpts = glob.glob(os.path.join(ckpt_dir, '*.ckpt'))
    # nothing to load, continue with fresh params
    if len(ckpts) == 0:
        return -1, None
    ckpts = map(lambda ckpt: (
        int(ckpt.split('.')[1]),
        ckpt), ckpts)
    # get most recent checkpoint
    epoch, ckpt_path = sorted(ckpts)[-1]
    return epoch, ckpt_path

def attempt_load_model(model, checkpoint_dir=None, checkpoint_path=None):
    assert checkpoint_dir or checkpoint_path

    if checkpoint_dir:
        epoch, checkpoint_path = get_latest_ckpt(checkpoint_dir)
    else:
        epoch = int(checkpoint_path.split('.')[-2])

    if checkpoint_path:
        model.load_state_dict(torch.load(checkpoint_path))
        print('Load from %s sucessful!' % checkpoint_path)
        return model, epoch + 1
    else:
        return model, 0

class SeqModel(nn.Module):
    def __init__(
        self,
        src_vocab_size,
        tgt_vocab_size,
        pad_id_src,
        pad_id_tgt,
        config=None,
    ):
        """Initialize model."""
        super(SeqModel, self).__init__()
        self.src_vocab_size = src_vocab_size
        self.tgt_vocab_size = tgt_vocab_size
        self.pad_id_src = pad_id_src
        self.pad_id_tgt = pad_id_tgt
        self.batch_size = config['data']['batch_size']
        self.config = config
        self.options = config['model']
        self.model_type = config['model']['model_type']

        self.src_embedding = nn.Embedding(
            self.src_vocab_size,
            self.options['emb_dim'],
            self.pad_id_src)

        if self.config['data']['share_vocab']:
            self.tgt_embedding = self.src_embedding
        else:
            self.tgt_embedding = nn.Embedding(
                self.tgt_vocab_size,
                self.options['emb_dim'],
                self.pad_id_tgt)

        if self.options['encoder'] == 'lstm':
            self.encoder = encoders.LSTMEncoder(
                self.options['emb_dim'],
                self.options['src_hidden_dim'],
                self.options['src_layers'],
                self.options['bidirectional'],
                self.options['dropout'])
            self.ctx_bridge = nn.Linear(
                self.options['src_hidden_dim'],
                self.options['tgt_hidden_dim'])

        else:
            raise NotImplementedError('unknown encoder type')

        # # # # # #  # # # # # #  # # # # #  NEW STUFF FROM STD SEQ2SEQ

        if self.model_type == 'delete':
            self.attribute_embedding = nn.Embedding(
                num_embeddings=2,
                embedding_dim=self.options['emb_dim'])
            attr_size = self.options['emb_dim']

        elif self.model_type == 'delete_retrieve':
            self.attribute_encoder = encoders.LSTMEncoder(
                self.options['emb_dim'],
                self.options['src_hidden_dim'],
                self.options['src_layers'],
                self.options['bidirectional'],
                self.options['dropout'],
                pack=False)
            attr_size = self.options['src_hidden_dim']

        elif self.model_type == 'seq2seq':
            attr_size = 0

        else:
            raise NotImplementedError('unknown model type')

        self.c_bridge = nn.Linear(
            attr_size + self.options['src_hidden_dim'],
            self.options['tgt_hidden_dim'])
        self.h_bridge = nn.Linear(
            attr_size + self.options['src_hidden_dim'],
            self.options['tgt_hidden_dim'])

        # # # # # #  # # # # # #  # # # # # END NEW STUFF

        self.decoder = decoders.StackedAttentionLSTM(config=config)

        self.output_projection = nn.Linear(
            self.options['tgt_hidden_dim'],
            tgt_vocab_size)

        self.softmax = nn.Softmax(dim=-1)

        self.init_weights()

    def init_weights(self):
        """Initialize weights."""
        initrange = 0.1
        self.src_embedding.weight.data.uniform_(-initrange, initrange)
        self.tgt_embedding.weight.data.uniform_(-initrange, initrange)
        self.h_bridge.bias.data.fill_(0)
        self.c_bridge.bias.data.fill_(0)
        self.output_projection.bias.data.fill_(0)

    def forward(self, input_src, input_tgt, srcmask, srclens, input_attr, attrlens, attrmask):
        src_emb = self.src_embedding(input_src)

        srcmask = (1-srcmask).byte()

        src_outputs, (src_h_t, src_c_t) = self.encoder(src_emb, srclens, srcmask)

        if self.options['bidirectional']:
            h_t = torch.cat((src_h_t[-1], src_h_t[-2]), 1)
            c_t = torch.cat((src_c_t[-1], src_c_t[-2]), 1)
        else:
            h_t = src_h_t[-1]
            c_t = src_c_t[-1]

        src_outputs = self.ctx_bridge(src_outputs)

        # # # #  # # # #  # #  # # # # # # #  # # seq2seq diff
        # join attribute with h/c then bridge 'em
        # TODO -- put this stuff in a method, overlaps w/above

        if self.model_type == 'delete':
            # just do h i guess?
            a_ht = self.attribute_embedding(input_attr)
            c_t = torch.cat((c_t, a_ht), -1)
            h_t = torch.cat((h_t, a_ht), -1)

        elif self.model_type == 'delete_retrieve':
            attr_emb = self.src_embedding(input_attr)
            _, (a_ht, a_ct) = self.attribute_encoder(attr_emb, attrlens, attrmask)
            if self.options['bidirectional']:
                a_ht = torch.cat((a_ht[-1], a_ht[-2]), 1)
                a_ct = torch.cat((a_ct[-1], a_ct[-2]), 1)
            else:
                a_ht = a_ht[-1]
                a_ct = a_ct[-1]

            h_t = torch.cat((h_t, a_ht), -1)
            c_t = torch.cat((c_t, a_ct), -1)

        c_t = self.c_bridge(c_t)
        h_t = self.h_bridge(h_t)

        # # # #  # # # #  # #  # # # # # # #  # # end diff

        tgt_emb = self.tgt_embedding(input_tgt)
        tgt_outputs, (_, _) = self.decoder(
            tgt_emb,
            (h_t, c_t),
            src_outputs,
            srcmask)

        tgt_outputs_reshape = tgt_outputs.contiguous().view(
            tgt_outputs.size()[0] * tgt_outputs.size()[1],
            tgt_outputs.size()[2])
        decoder_logit = self.output_projection(tgt_outputs_reshape)
        decoder_logit = decoder_logit.view(
            tgt_outputs.size()[0],
            tgt_outputs.size()[1],
            decoder_logit.size()[1])

        probs = self.softmax(decoder_logit)

        return decoder_logit, probs

    def count_params(self):
        n_params = 0
        for param in self.parameters():
            n_params += np.prod(param.data.cpu().numpy().shape)
        return n_params


text:
"""
DBSCAN over the batch across time

"""

import torch
import numpy as np
from einops import rearrange,repeat
from sklearn import cluster as sk_cluster

from .cluster_update_impl import init_clusters
from .centroid_update_impl import update_ecentroids

def pwd_silly(a,b):
    c,ta,s,h,w,ps,ps = a.shape
    ta,tb = a.shape[1],b.shape[1]
    dists = torch.zeros((tb,ta,s,h,w)).to(a.device)
    for ti in range(ta):
        for tj in range(tb):
            sq_delta = (a[:,ti]-b[:,tj])**2
            sq_delta = torch.sum(sq_delta,axis=(0,4,5))
            dists[tj,ti] = sq_delta
    return dists

def run_dbscan_scikit(burst,indices,K,ps,km_offset,eps,minPts):

    # -- unpack --
    device = burst.device
    c,t,h,w = burst.shape
    two,t,s,h,w = indices.shape
    cluster_ps = 11

    # -- create centroids for each frame --
    clusters,sizes = init_clusters(t,t,s,h,w,device)
    eburst = update_ecentroids(burst,indices,clusters,sizes,cluster_ps)
    eburst = eburst.cpu().numpy()
    sizes = torch.zeros_like(sizes)

    # -- run dbscan for each prop --
    for si in range(s):
        for hi in range(h):
            for wi in range(w):

                # -- extract sample --
                pburst = eburst[:,:,si,hi,wi,:,:]
                pburst = rearrange(pburst,'c t p1 p2 -> t (c p1 p2)')

                # -- run dbscan --
                sk_dbscan = sk_cluster.DBSCAN(eps=eps,min_samples=minPts)
                # sk_dbscan = sk_cluster.OPTICS(min_samples=minPts)
                sk_dbscan = sk_dbscan.fit(pburst)

                # -- format output --
                cids = sk_dbscan.labels_
                cids = torch.IntTensor(cids).to(device)
                # print("c",cids)

                # -- remove -1 indices --
                max_cid = cids.max().item()+1
                for c,cid in enumerate(cids):
                    if cid == -1:
                        cids[c] = max_cid
                        max_cid += 1
                # print("b",cids)

                # -- update --
                clusters[:,si,hi,wi] = cids
                sizes[cids.type(torch.long),si,hi,wi] += 1
                # print(sizes[:,si,hi,wi])

    # -- to device --
    centroids = update_ecentroids(burst,indices,clusters,sizes,ps)
    # centroids = torch.FloatTensor(centroids).to(device)

    return None,clusters,sizes,centroids

def run_dbscan(burst,indices,K,ps,km_offset,eps,minPts):

    # -- unpack --
    device = burst.device
    c,t,h,w = burst.shape
    two,t,s,h,w = blocks.shape
    clusters = torch.ByteTensor(torch.zeros((t,s,h,w)))
    nframes = t

    # -- dbscan params --
    eps = 1e-1
    minPts = 2

    # -- start dbscan --
    nclusters = 0
    for p in range(nframes):

        # -- get neighbors --
        neighbors = query_neighbors(burst,indices,p,eps)

        # -- filter min points --
        mask = filter_min_pts(neighbors,p,clusters,sizes,minPts)

        # -- update clusters --
        nclusters += 1

        # -- label initial --
        clusters,sizes = update_label(nclusters,p,clusters,sizes)

        for q in neighbors:
            if q == p: continue

            # -- update noise labels --
            clusters,sizes = update_noise_label(nclusters,q,clusters,sizes)

            # -- mask out already labeled points --
            mask = rm_labeled_points(q,clusters,sizes)

            # -- label non-masked points as q --
            clusters,sizes = update_label(nclusters,q,clusters,sizes,mask)

            # -- compute neighbors for undef points --
            q_neighbors = query_neighbors(burst,indices,q,eps)

            # -- filter min points --
            clusters,sizes = filter_min_pts(q_neighbors,q,clusters,sizes,minPts,mask)

            # -- update set --

    return dists,clusters,sizes,centroids

ext: py

reason: import numpy


text:
import numpy as np

class LayerDense():

  def __init__(self,n_neurons,function_of_activation,imput_shape):
    self.n_neurons=n_neurons
    self.function_of_activation=function_of_activation
    self.imput_shape=imput_shape
    self.bias=np.random.randn(n_neurons,1)
    self.weights=np.random.randn(self.n_neurons,self.imput_shape[0])
    self.linear_convination=None
    self.activation=None

  def foward_layer_dense(self,features):
    if self.imput_shape[0]==1:
      self.linear_convination=self.bias + self.weights*features
    else:
      self.linear_convination=self.bias + self.weights@features
    self.activation=self.function_of_activation(self.linear_convination)
    return self.activation

  def bacward_layer_dense(self, learning_ratio, gradient_next_layer,weights_next_layer=0,activation_previous_layer=0):
    if type(weights_next_layer)==int and type(activation_previous_layer)==int:
      this_layer_gradient=np.mean(gradient_next_layer*self.function_of_activation(self.linear_convination, derivative=True))
      self.bias = self.bias -learning_ratio*this_layer_gradient
      self.weights=self.weights-learning_ratio*this_layer_gradient*np.mean(activation_previous_layer)
      return this_layer_gradient
    else :
      this_layer_gradient=np.mean(gradient_next_layer*weights_next_layer@self.function_of_activation(self.linear_convination, derivative=True))
      self.bias = self.bias -learning_ratio*this_layer_gradient
      self.weights=self.weights-learning_ratio*this_layer_gradient*np.mean(activation_previous_layer)
      return this_layer_gradient

max_stars_repo_name: Mitchell-Mirano/allison

max_issues_repo_name: dfotalvaroz/Allison

max_forks_repo_name: dfotalvaroz/Allison


text:
import numpy as np
import pandas as pd

np.random.seed(0)
from aif360.metrics import ClassificationMetric
from aif360.datasets import *
from sklearn.metrics import accuracy_score
pd.options.mode.chained_assignment = None  # default='warn'

def classifier_score(pred_fun, metrics, df, protected_label, privileged_value, output_name, threshold, output_index):
    privilged_population = df[df[protected_label] == privileged_value]
    unprivilged_population = df[df[protected_label] != privileged_value]

    if (output_index):
        orig_privileged_scores = pred_fun(privilged_population.drop(columns=[output_name]).values)[:, output_index]
        orig_unprivileged_scores = pred_fun(unprivilged_population.drop(columns=[output_name]).values)[:, output_index]
    else:
        orig_privileged_scores = pred_fun(privilged_population.drop(columns=[output_name]).values)[:, 0]
        orig_unprivileged_scores = pred_fun(unprivilged_population.drop(columns=[output_name]).values)[:, 0]

    orig_scores = np.vstack([orig_privileged_scores.reshape(-1, 1),
                             orig_unprivileged_scores.reshape(-1, 1)])

    orig_scores[orig_scores <= 0.5] = 0
    orig_scores[orig_scores > 0.5] = 1

    original_privileged_accuracy = accuracy_from_scores(privilged_population[output_name], orig_privileged_scores,
                                                        threshold)
    original_unprivileged_accuracy = accuracy_from_scores(unprivilged_population[output_name], orig_unprivileged_scores,
                                                          threshold)

    original_accuracy = (len(privilged_population) * original_privileged_accuracy + len(
        unprivilged_population) * original_unprivileged_accuracy) / len(df)

    pre_transform_df = pd.concat([privilged_population, unprivilged_population])
    pre_transform_df[output_name] = np.hstack([orig_privileged_scores, orig_unprivileged_scores])
    pre_transform_df[output_name] = clip_at_threshold(pre_transform_df[output_name], 0.5)

    orig_dataset = BinaryLabelDataset(df=pd.concat([privilged_population, unprivilged_population]),
                                      favorable_label=1.0,
                                      unfavorable_label=0.0,
                                      label_names=[output_name],
                                      protected_attribute_names=[protected_label],
                                      privileged_protected_attributes=[privileged_value])

    pre_trans_dataset = BinaryLabelDataset(df=pre_transform_df,
                                           favorable_label=1.0,
                                           unfavorable_label=0.0,
                                           label_names=[output_name],
                                           protected_attribute_names=[protected_label],
                                           privileged_protected_attributes=[privileged_value])

    privileged_groups = {}
    privileged_groups[protected_label] = privileged_value
    unprivileged_groups = {}
    unprivileged_groups[protected_label] = 1 - privileged_value

    pre_trans_dataset.scores = pre_transform_df[output_name]
    dataset = ClassificationMetric(orig_dataset,
                                   pre_trans_dataset,
                                   privileged_groups=[privileged_groups],
                                   unprivileged_groups=[unprivileged_groups])
    return np.asanyarray([getattr(dataset, metric)() for metric in metrics])

def clip_at_threshold(y, threshold=0.5):
    y[y >= threshold] = 1
    y[y < threshold] = 0
    return y

def accuracy_from_scores(y_true, y_score, threshold):
    y_pred = np.copy(y_score)
    y_pred[y_pred <= threshold] = 0
    y_pred[y_pred > threshold] = 1
    return accuracy_score(y_true, y_pred)


text:
import torch
import torch.nn.functional as F
import numpy as np
import os
import glob
from tqdm import tqdm
import SimpleITK as sitk
from utils.misc import get_spacing

os.environ['CUDA_VISIBLE_DEVICES'] = "1"

DATABASE = 'VascuSynth3/'
#
args = {
    'root'     : './dataset/' + DATABASE,
    'test_path': './dataset/' + DATABASE + 'test/',
    'pred_path': 'assets/' + 'VascuSynth3/',
    'img_size' : 512
}

if not os.path.exists(args['pred_path']):
    os.makedirs(args['pred_path'])

def rescale(img):
    w, h = img.size
    min_len = min(w, h)
    new_w, new_h = min_len, min_len
    scale_w = (w - new_w) // 2
    scale_h = (h - new_h) // 2
    box = (scale_w, scale_h, scale_w + new_w, scale_h + new_h)
    img = img.crop(box)
    return img

def load_3d():
    test_images = []
    test_labels = []
    for file in glob.glob(os.path.join(args['test_path'], 'images', '*.mha')):
        basename = os.path.basename(file)
        file_name = basename[:-8]
        image_name = os.path.join(args['test_path'], 'images', basename)
        label_name = os.path.join(args['test_path'], 'label', file_name + 'gt.mha')
        test_images.append(image_name)
        test_labels.append(label_name)
    return test_images, test_labels

def load_net():
    net = torch.load('/home/imed/Research/Attention/checkpoint/model.pkl')
    return net

def save_prediction(pred, filename='', spacing=None):
    pred = torch.argmax(pred, dim=1)
    save_path = args['pred_path'] + 'pred/'
    if not os.path.exists(save_path):
        os.makedirs(save_path)
        print("Make dirs success!")
    # for MSELoss()
    mask = (pred.data.cpu().numpy() * 255).astype(np.uint8)

    # thresholding
    # mask[mask >= 100] = 255
    # mask[mask < 100] = 0

    # mask = (mask.squeeze(0)).squeeze(0)  # 3D numpy array
    mask = mask.squeeze(0)  # for CE Loss
    # image = nib.Nifti1Image(np.int32(mask), affine)
    # nib.save(image, save_path + filename + ".nii.gz")
    mask = sitk.GetImageFromArray(mask)
    # if spacing is not None:
    #     mask.SetSpacing(spacing)
    sitk.WriteImage(mask, os.path.join(save_path + filename + ".mha"))

def save_probability(pred, label, filename=""):
    save_path = args['pred_path'] + 'pred/'
    if not os.path.exists(save_path):
        os.makedirs(save_path)
        print("Make dirs success!")
    # # for MSELoss()
    # mask = (pred.data.cpu().numpy() * 255)  # .astype(np.uint8)
    #
    # mask = mask.squeeze(0)
    # class0 = mask[0, :, :, :]
    # class1 = mask[1, :, :, :]
    # label = label / 255
    # class0 = class0 * label
    # class1 = class1 * label
    #
    # probability = class0 + class1

    probability = F.softmax(pred, dim=1)
    probability.squeeze_(0)
    class0 = probability[0, :, :, :]
    class1 = probability[1, :, :, :]
    class0 = sitk.GetImageFromArray(class0)
    class1 = sitk.GetImageFromArray(class1)
    sitk.WriteImage(class1, os.path.join(save_path + filename + "class1.mha"))

def save_label(label, index, spacing=None):
    label_path = args['pred_path'] + 'label/'
    if not os.path.exists(label_path):
        os.makedirs(label_path)
    label = sitk.GetImageFromArray(label)
    if spacing is not None:
        label.SetSpacing(spacing)
    sitk.WriteImage(label, os.path.join(label_path, index + ".mha"))

def predict():
    net = load_net()
    images, labels = load_3d()
    with torch.no_grad():
        net.eval()
        for i in tqdm(range(len(images))):
            name_list = images[i].split('/')
            index = name_list[-1][:-4]
            image = sitk.ReadImage(images[i])
            image = sitk.GetArrayFromImage(image).astype(np.float32)
            image = image / 255
            label = sitk.ReadImage(labels[i])
            label = sitk.GetArrayFromImage(label).astype(np.int64)
            # label = label / 255
            # VascuSynth
            # image = image[2:98, 2:98, 2:98]
            # label = label[2:98, 2:98, 2:98]
            save_label(label, index)
            # if cuda
            image = torch.from_numpy(np.ascontiguousarray(image)).unsqueeze(0).unsqueeze(0)
            image = image.cuda()
            output = net(image)
            save_prediction(output, filename=index + '_pred', spacing=None)

if __name__ == '__main__':
    predict()

max_stars_repo_name: iMED-Lab/CS-Net

max_issues_repo_name: clguo/CS-Net

max_forks_repo_name: clguo/CS-Net


text:
# Copyright 2016 The Johns Hopkins University Applied Physics Laboratory
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import six
from abc import ABCMeta, abstractmethod
import numpy as np
from PIL import Image

@six.add_metaclass(ABCMeta)
class TileProcessor(object):
    def __init__(self):
        """
        A class to implement a tile processor which outputs a list of file handles for uploading

        Args:
        """
        self.parameters = None

    @abstractmethod
    def setup(self, parameters):
        """
        Method to initialize the tile processor based on custom parameters from the configuration file

        e.g. Open a multi-page tiff, connect to a database, etc.

        Args:
            parameters (dict): Parameters for the dataset to be processed

        Returns:
            None
        """
        return NotImplemented

    @abstractmethod
    def process(self, file_path, x_index, y_index, z_index, t_index=None):
        """
        Method to take a file path and tile indices and return a file handle to the correct data

        Args:
            file_path(str): An absolute file path for the specified tile
            x_index(int): The tile index in the X dimension
            y_index(int): The tile index in the Y dimension
            z_index(int): The tile index in the Z dimension
            t_index(int): The time index

        Returns:
            (io.BufferedReader): A file handle for the specified tile

        """
        return NotImplemented

class TestTileProcessor(TileProcessor):
    """Example processor for unit tests"""

    def setup(self, parameters):
        """
        Method to initialize the tile processor based on custom parameters from the configuration file

        e.g. Open a multi-page tiff, connect to a database, etc.

        Args:
            parameters (dict): Parameters for the dataset to be processed

        Returns:
            None
        """
        self.parameters = parameters

    def process(self, file_path, x_index, y_index, z_index, t_index=None):
        """
        Method to take a file path and tile indices and return a file handle to the correct data

        Always just returns the handle to the file

        Args:
            file_path(str): An absolute file path for the specified tile
            x_index(int): The tile index in the X dimension
            y_index(int): The tile index in the Y dimension
            z_index(int): The tile index in the Z dimension
            t_index(int): The time index

        Returns:
            (io.BufferedReader): A file handle for the specified tile

        """
        return open(file_path, 'rb')

class TestRandomTileProcessor(TileProcessor):
    """Example processor for scale tests"""

    def setup(self, parameters):
        """
        Method to initialize the tile processor based on custom parameters from the configuration file

        e.g. Open a multi-page tiff, connect to a database, etc.

        Args:
            parameters (dict): Parameters for the dataset to be processed

        Returns:
            None
        """
        self.parameters = parameters

    def process(self, file_path, x_index, y_index, z_index, t_index=None):
        """
        Generate a random tile

        Args:
            file_path(str): An absolute file path for the specified tile
            x_index(int): The tile index in the X dimension
            y_index(int): The tile index in the Y dimension
            z_index(int): The tile index in the Z dimension
            t_index(int): The time index

        Returns:
            (io.BufferedReader): A file handle for the specified tile

        """
        tile = np.random.randint(1, 254, size=(self.parameters["ingest_job"]["tile_size"]["y"],
                                               self.parameters["ingest_job"]["tile_size"]["x"]), dtype=np.uint8)
        tile_data = Image.fromarray(tile)
        output = six.BytesIO()
        tile_data.save(output, format="TIFF")

        return output


text:
import pandas as pd
import numpy as np

df = pd.read_csv('all_countries.csv')
#df.fillna('-', inplace=True)

main_pf = ['pf_rol', 'pf_ss', 'pf_movement', 'pf_religion', 'pf_association', 'pf_expression', 'pf_identity']
main_ef = ['ef_government', 'ef_legal', 'ef_money', 'ef_trade', 'ef_regulation']

other_cat_pf = ['pf_rol_procedural', 'pf_rol_civil', 'pf_rol_criminal', 'pf_ss_homicide',
                'pf_ss_disappearances', 'pf_ss_women', 'pf_movement_domestic', 'pf_movement_foreign',
                'pf_movement_women', 'pf_religion_freedom', 'pf_religion_repression',
                'pf_religion_harassment', 'pf_religion_restrictions', 'pf_association_entry',
                'pf_association_assembly', 'pf_association_estopparties', 'pf_association_opposition',
                'pf_association_civilrepression', 'pf_expression_killed', 'pf_expression_jailed',
                'pf_expression_media', 'pf_expression_cable', 'pf_expression_newspapers', 'pf_expression_control',
                'pf_identity_legal', 'pf_identity_sex', 'pf_identity_divorce']

other_cat_ef = ['ef_government_consumption', 'ef_government_transfers',
                'ef_government_enterprises', 'ef_government_tax', 'ef_government_soa',
                'ef_legal_judicial', 'ef_legal_courts', 'ef_legal_protection', 'ef_legal_military',
                'ef_legal_integrity', 'ef_legal_enforcement', 'ef_legal_regulatory',
                'ef_legal_police', 'ef_money_growth', 'ef_money_sd', 'ef_money_inflation',
                'ef_money_currency', 'ef_trade_tariffs', 'ef_trade_regulatory', 'ef_trade_black',
                'ef_trade_movement', 'ef_regulation_credit', 'ef_regulation_labor', 'ef_regulation_business']

all_pf = ['pf_rol', 'pf_rol_procedural', 'pf_rol_civil', 'pf_rol_criminal', 'pf_ss', 'pf_ss_homicide',
          'pf_ss_disappearances', 'pf_ss_women', 'pf_movement', 'pf_movement_domestic',
          'pf_movement_foreign', 'pf_movement_women', 'pf_religion', 'pf_religion_freedom',
          'pf_religion_repression', 'pf_religion_harassment', 'pf_religion_restrictions', 'pf_association',
          'pf_association_entry', 'pf_association_assembly', 'pf_association_estopparties', 'pf_association_opposition',
          'pf_association_civilrepression', 'pf_expression', 'pf_expression_killed', 'pf_expression_jailed',
          'pf_expression_media', 'pf_expression_cable', 'pf_expression_newspapers', 'pf_expression_control',
          'pf_identity', 'pf_identity_legal', 'pf_identity_sex', 'pf_identity_divorce']

all_ef = ['ef_government', 'ef_government_consumption', 'ef_government_transfers', 'ef_government_enterprises',
          'ef_government_tax', 'ef_government_soa', 'ef_legal', 'ef_legal_judicial', 'ef_legal_courts',
          'ef_legal_protection', 'ef_legal_military', 'ef_legal_integrity', 'ef_legal_enforcement',
          'ef_legal_regulatory', 'ef_legal_police', 'ef_money', 'ef_money_growth', 'ef_money_sd', 'ef_money_inflation',
          'ef_money_currency', 'ef_trade', 'ef_trade_tariffs', 'ef_trade_regulatory', 'ef_trade_black',
          'ef_trade_movement', 'ef_regulation', 'ef_regulation_credit', 'ef_regulation_labor', 'ef_regulation_business']

country_file = ['Albania', 'Algeria', 'Angola', 'Argentina', 'Armenia', 'Australia', 'Austria',
                'Azerbaijan', 'Bahamas', 'Bahrain', 'Bangladesh', 'Barbados', 'Belarus',
                'Belgium', 'Belize', 'Benin', 'Bhutan', 'Bolivia', 'Bosnia',
                'Botswana', 'Brazil', 'Brunei', 'Bulgaria', 'Burkina', 'Burundi',
                'Cabo', 'Cambodia', 'Cameroon', 'Canada', 'Central', 'Chad', 'Chile', 'China', 'Colombia',
                'Congodem', 'Congorep', 'Costa', 'Cote', 'Croatia', 'Cyprus', 'Czech', 'Denmark',
                'Dominican', 'Ecuador', 'Egypt', 'Salvador', 'Estonia', 'Eswatini', 'Ethiopia', 'Fiji',
                'Finland', 'France', 'Gabon', 'Gambia', 'Georgia', 'Germany', 'Ghana',
                'Greece', 'Guatemala', 'Guinea', 'Bissau', 'Guyana', 'Haiti', 'Honduras', 'Hong',
                'Hungary', 'Iceland', 'India', 'Indonesia', 'Iran', 'Iraq', 'Ireland', 'Israel', 'Italy',
                'Jamaica', 'Japan', 'Jordan', 'Kazakhstan', 'Kenya', 'Korea', 'Kuwait', 'Kyrgyz',
                'Lao', 'Latvia', 'Lebanon', 'Lesotho', 'Liberia', 'Libya', 'Lithuania', 'Luxembourg',
                'Madagascar', 'Malawi', 'Malaysia', 'Mali', 'Malta', 'Mauritania', 'Mauritius', 'Mexico',
                'Moldova', 'Mongolia', 'Montenegro', 'Morocco', 'Mozambique', 'Myanmar', 'Namibia', 'Nepal',
                'Netherlands', 'Zealand', 'Nicaragua', 'Niger', 'Nigeria', 'Macedonia', 'Norway', 'Oman',
                'Pakistan', 'Panama', 'Papua', 'Paraguay', 'Peru', 'Philippines', 'Poland', 'Portugal',
                'Qatar', 'Romania', 'Russian', 'Rwanda', 'Saudi', 'Senegal', 'Serbia', 'Seychelles',
                'Leone', 'Singapore', 'Slovak', 'Slovenia', 'Africa', 'Spain', 'Sri Lanka', 'Sudan', 'Suriname',
                'Sweden', 'Switzerland', 'Syrian', 'Taiwan', 'Tajikistan', 'Tanzania', 'Thailand', 'Timor',
                'Togo', 'Trinidad', 'Tunisia', 'Turkey', 'Uganda', 'Ukraine', 'Emirates', 'Kingdom', 'UnitedStates',
                'Uruguay', 'Venezuela', 'Vietnam', 'Yemen', 'Zambia', 'Zimbabwe']

country = df['countries'].unique()
country_name = [i.upper() for i in country]
region = [i.upper() for i in df.loc[df['year'] == 2018, 'region']]
ranking2018 = [int(i) for i in df.loc[df['year'] == 2018, 'hf_rank']]
hfscore2018 = [str('{:.2f}'.format(i)) for i in df.loc[df['year'] == 2018, 'hf_score']] # two decimal points
rankingpf = [(str(int(i))+'/162') for i in df.loc[df['year'] == 2018, 'pf_rank']]
rankingef = [(str(int(i))+'/162') for i in df.loc[df['year'] == 2018, 'ef_rank']]
pfscore2018 = [str('{:.2f}'.format(i)) for i in df.loc[df['year'] == 2018, 'pf_score']] # two decimal points
efscore2018 = [str('{:.2f}'.format(i)) for i in df.loc[df['year'] == 2018, 'ef_score']] # two decimal points

# personal and economic freedom categories
list_score_pf_main = []
list_score_ef_main = []
list_score_other_pf_main = []
list_score_other_ef_main = []
for i in country:
    pf = []
    ef = []
    other_pf = []
    other_ef = []

    for j in main_pf:
        x = df.loc[(df['year'] == 2018) & (df['countries'] == i), j]
        pf.append(float(x))

    for k in main_ef:
        x = df.loc[(df['year'] == 2018) & (df['countries'] == i), k]
        ef.append(float(x))

    for l in other_cat_pf:
        x = df.loc[(df['year'] == 2018) & (df['countries'] == i), l]
        other_pf.append(float(x))

    for m in other_cat_ef:
        x = df.loc[(df['year'] == 2018) & (df['countries'] == i), m]
        other_ef.append(float(x))

    list_score_ef_main.append(ef)
    list_score_pf_main.append(pf)
    list_score_other_ef_main.append(other_ef)
    list_score_other_pf_main.append(other_pf)

# create list with final ef and pf values (main categories)
final_pf_main = []
for i in list_score_pf_main:
    a = str(i[0]) + '\n'*5 + str(i[1]) + '\n'*5 + str(i[2]) + '\n'*5 + str(i[3]) + '\n'*6 + str(i[4]) + '\n'*7 \
        + str(i[5]) + '\n'*8 + str(i[6])
    final_pf_main.append(a)

final_ef_main = []
for j in list_score_ef_main:
    a = str(j[0]) + '\n'*7 + str(j[1]) + '\n'*10 + str(j[2]) + '\n'*6 + str(j[3]) + '\n'*6 + str(j[4])
    final_ef_main.append(a)

# create a list with final ef and pf value (other categories)
final_other_ef_main = []
for i in list_score_other_ef_main:
    a = '\n' + str(i[0]) + '\n' + str(i[1]) + '\n' + str(i[2]) + '\n' + str(i[3]) + '\n' + str(i[4]) + '\n'*3 + \
        str(i[5]) + '\n' + str(i[6]) + '\n' + str(i[7]) + '\n' + str(i[8]) + '\n' + str(i[9]) + '\n' + str(i[10]) + \
        '\n' + str(i[11]) + '\n' + str(i[12]) + '\n'*3 + \
        str(i[13]) + '\n' + str(i[14]) + '\n' + str(i[15]) + '\n' + str(i[16]) + '\n'*3 + \
        str(i[17]) + '\n' + str(i[18]) + '\n' + str(i[19]) + '\n' + str(i[20]) + '\n'*3 + \
        str(i[21]) + '\n' + str(i[22]) + '\n' + str(i[23])
    final_other_ef_main.append(a)

final_other_pf_main = []
for i in list_score_other_pf_main:
    a = '\n' + str(i[0]) + '\n' + str(i[1]) + '\n' + str(i[2]) + '\n'*3 + \
        str(i[3]) + '\n' + str(i[4]) + '\n' + str(i[5]) + '\n'*3 + \
        str(i[6]) + '\n' + str(i[7]) + '\n' + str(i[8]) + '\n'*3 + \
        str(i[9]) + '\n' + str(i[10]) + '\n' + str(i[11]) + '\n' + str(i[12]) + '\n'*3 + \
        str(i[13]) + '\n' + str(i[14]) + '\n' + str(i[15]) + '\n' + str(i[16]) + '\n' + str(i[17]) + '\n'*3 + \
        str(i[18]) + '\n' + str(i[19]) + '\n' + str(i[20]) + '\n' + str(i[21]) + '\n' + str(i[22]) + '\n' + str(i[23]) + '\n'*3 + \
        str(i[24]) + '\n' + str(i[25]) + '\n' + str(i[26])
    final_other_pf_main.append(a)

# graph pf and ef
for co in range(len(country)):
    pf = []
    ef = []

    for j in all_pf:
        x = df.loc[(df['year'] == 2018) & (df['countries'] == country[co]), j]
        pf.append(float(x))

    for k in all_ef:
        x = df.loc[(df['year'] == 2018) & (df['countries'] == country[co]), k]
        ef.append(float(x))

    # personal freedom
    pos = [4, 9, 14, 20, 27, 35]
    acc = 0
    for i in range(len(pos)):
        pf.insert(pos[i], 0)
        acc += 1
    w = pd.DataFrame(pf)
    w.to_csv('/Users/glosophy/Dropbox/Human Freedom Index/2020/Data/GraphPF/{}.csv'.format(country_file[co]),
             index=False, header=False)

    # economic freedom
    pos = [6, 15, 20, 25]
    acc = 0
    for i in range(len(pos)):
        ef.insert(pos[i] + acc, 0)
        acc += 1
    w = pd.DataFrame(ef)
    w.to_csv('/Users/glosophy/Dropbox/Human Freedom Index/2020/Data/GraphEF/{}.csv'.format(country_file[co]),
             index=False, header=False)

# human freedom chart - order: country, world, region
selected = pd.read_csv('selected_countries.csv')
# selected.fillna('-', inplace=True)
# region1 = pd.read_csv('CAT_hf_score_NO_GENDER.csv')

# world average
world_avg = []
for i in selected['year'].unique():
    mean = selected.loc[selected['year'] == i, 'hf_score_NO_GENDER'].mean()
    world_avg.append(mean)

world_avg = world_avg[::-1]

# score countries
countries_score = []
region_score = []
for i in df['countries'].unique():
    country_hf = []

    cat_by_region = selected.groupby(['year', 'region'])['hf_score_NO_GENDER'].mean().reset_index()
    cat_pivot = cat_by_region.pivot(index='year', columns='region', values='hf_score_NO_GENDER')
    reg = df.loc[(df['year'] == 2018) & (df['countries'] == i), 'region'].values[0]
    reg_score = cat_pivot[str(reg)].to_list()
    region_score.append(reg_score)

    for j in df['year'].unique():
        hf_score = df.loc[(df['year'] == j) & (df['countries'] == i), 'hf_score_NO_GENDER']
        country_hf.append(float(hf_score))

    country_hf = country_hf[::-1]
    countries_score.append(country_hf)

for i in range(len(country)):
    hf_graph = {'hf':countries_score[i],
                'world': world_avg,
                'reg':region_score[i]}
    graph_df = pd.DataFrame(hf_graph)
    graph_df.to_csv('/Users/glosophy/Dropbox/Human Freedom Index/2020/Data/GraphHF/{}.csv'.format(country_file[i]),
                    index=False, header=False)

# ranking graph
for i in range(len(country)):
    rank_country = []
    for j in df['year'].unique():
        ranking = df.loc[(df['year'] == j) & (df['countries'] == country[i]), 'hf_rank']
        rank_country.append(float(ranking))
    rank_country = rank_country[::-1]
    w = pd.DataFrame(rank_country)
    w.to_csv('/Users/glosophy/Dropbox/Human Freedom Index/2020/Data/GraphRank/{}.csv'.format(country_file[i]),
             index=False, header=False)

# main years second page
list_score_pf_main_page2 = []
list_score_other_pf_main_page2 = []
for k in df['year'].unique():
    pf = []
    other_pf = []

    for i in country:
        years_main = []
        years_other = []
        for j in main_pf:
            x = df.loc[(df['year'] == k) & (df['countries'] == i), j]
            years_main.append(float(x))

        for l in other_cat_pf:
            x = df.loc[(df['year'] == k) & (df['countries'] == i), l]
            years_other.append(float(x))

        pf.append(years_main)
        other_pf.append(years_other)

    years_pf_pf = []
    for m in pf:
        a = str(m[0]) + '\n'*5 + str(m[1]) + '\n'*5 + str(m[2]) + '\n'*5 + str(m[3]) + '\n'*6 + str(m[4]) + '\n'*7 \
            + str(m[5]) + '\n'*8 + str(m[6])
        years_pf_pf.append(a)

    years_other_other = []
    for n in other_pf:
        b = '\n' * 1 + str(n[0]) + '\n' + str(n[1]) + '\n' + str(n[2]) + '\n' * 3 + \
            str(n[3]) + '\n' + str(n[4]) + '\n' + str(n[5]) + '\n' * 3 + \
            str(n[6]) + '\n' + str(n[7]) + '\n' + str(n[8]) + '\n' * 3 + \
            str(n[9]) + '\n' + str(n[10]) + '\n' + str(n[11]) + '\n' + str(n[12]) + '\n' * 3 + \
            str(n[13]) + '\n' + str(n[14]) + '\n' + str(n[15]) + '\n' + str(n[16]) + '\n' + str(n[17]) + '\n' * 3 + \
            str(n[18]) + '\n' + str(n[19]) + '\n' + str(n[20]) + '\n' + str(n[21]) + '\n' + str(n[22]) + '\n' + str(
            n[23]) + '\n' * 3 + str(n[24]) + '\n' + str(n[25]) + '\n' + str(n[26])
        years_other_other.append(b)

    list_score_pf_main_page2.append(years_pf_pf)
    list_score_other_pf_main_page2.append(years_other_other)

# score country by year
score_year = []
for k in df['year'].unique():
    score = []
    for i in country:
        x = df.loc[(df['year'] == k) & (df['countries'] == i), 'hf_score']
        x = float(x)
        score.append('{:.2f}'.format(x))
    score_year.append(score)

# ranking by year
rank_year = []
for k in df['year'].unique():
    rank = []
    for i in country:
        x = df.loc[(df['year'] == k) & (df['countries'] == i), 'hf_rank']
        x = float(x)
        rank.append('{:.2f}'.format(x))
    rank_year.append(rank)

# dscores:
dscore = []
for k in df['year'].unique():
    midstep = []
    for i in country:
        x = df.loc[(df['year'] == k) & (df['countries'] == i), 'pf_score'].values[0]
        y = df.loc[(df['year'] == k) & (df['countries'] == i), 'ef_score'].values[0]
        # x = float(x)
        # y = float(y)
        a = str('{:.2f}'.format(y)) + '\n' + str('{:.2f}'.format(x))
        midstep.append(a)
    dscore.append(midstep)

scores = []
for i in df['year'].unique():
    midstep = []
    for k in country:
        x = df.loc[(df['year'] == i) & (df['countries'] == k), 'hf_score'].values[0]
        midstep.append(str('{:.2f}'.format(float(x))))
    scores.append(midstep)

rank_years = []
for i in df['year'].unique():
    midstep2 = []
    for k in country:
        x = df.loc[(df['year'] == i) & (df['countries'] == k), 'hf_rank'].values[0]
        midstep2.append(str('{:.0f}'.format(float(x))))
    rank_years.append(midstep2)

# create dictionary with all variables
d = {'countryname': country_name,
     'country': country,
     'region': region,
     'ranking': ranking2018,
     'score': hfscore2018,
     'rankingpf': rankingpf,
     'scorepf': pfscore2018,
     'rankingef': rankingef,
     'scoreef': efscore2018,
     'listscorepfmain': final_pf_main,
     'listscorepf': final_other_pf_main,
     '%graphpf': ['/Users/guillermina/Dropbox/Human Freedom Index/2020/Data/GraphPF/{}.csv'.format(co) for co in country_file],
     'listscoreefmain': final_ef_main,
     'listscoreef': final_other_ef_main,
     '%graphef': ['/Users/guillermina/Dropbox/Human Freedom Index/2020/Data/GraphEF/{}.csv'.format(co) for co in country_file],
     '%graphscorehf': ['/Users/guillermina/Dropbox/Human Freedom Index/2020/Data/GraphHF/{}.csv'.format(co) for co in country_file],
     '%graphrankinghf': ['/Users/guillermina/Dropbox/Human Freedom Index/2020/Data/GraphRank/{}.csv'.format(co) for co in country_file],
     'yearsmain2018': list_score_pf_main_page2[0],
     'yearsmain2017': list_score_pf_main_page2[1],
     'yearsmain2016': list_score_pf_main_page2[2],
     'yearsmain2015': list_score_pf_main_page2[3],
     'yearsmain2014': list_score_pf_main_page2[4],
     'yearsmain2013': list_score_pf_main_page2[5],
     'yearsmain2012': list_score_pf_main_page2[6],
     'yearsmain2011': list_score_pf_main_page2[7],
     'yearsmain2010': list_score_pf_main_page2[8],
     'yearsmain2009': list_score_pf_main_page2[9],
     'yearsmain2008': list_score_pf_main_page2[10],
     'year2018': list_score_other_pf_main_page2[0],
     'year2017': list_score_other_pf_main_page2[1],
     'year2016': list_score_other_pf_main_page2[2],
     'year2015': list_score_other_pf_main_page2[3],
     'year2014': list_score_other_pf_main_page2[4],
     'year2013': list_score_other_pf_main_page2[5],
     'year2012': list_score_other_pf_main_page2[6],
     'year2011': list_score_other_pf_main_page2[7],
     'year2010': list_score_other_pf_main_page2[8],
     'year2009': list_score_other_pf_main_page2[9],
     'year2008': list_score_other_pf_main_page2[10],
     'dscore2018': dscore[0],
     'dscore2017': dscore[1],
     'dscore2016': dscore[2],
     'dscore2015': dscore[3],
     'dscore2014': dscore[4],
     'dscore2013': dscore[5],
     'dscore2012': dscore[6],
     'dscore2011': dscore[7],
     'dscore2010': dscore[8],
     'dscore2009': dscore[9],
     'dscore2008': dscore[10],
     'score2018': scores[0],
     'score2017': scores[1],
     'score2016': scores[2],
     'score2015': scores[3],
     'score2014': scores[4],
     'score2013': scores[5],
     'score2012': scores[6],
     'score2011': scores[7],
     'score2010': scores[8],
     'score2009': scores[9],
     'score2008': scores[10],
     'ranking2018': rank_years[0],
     'ranking2017': rank_years[1],
     'ranking2016': rank_years[2],
     'ranking2015': rank_years[3],
     'ranking2014': rank_years[4],
     'ranking2013': rank_years[5],
     'ranking2012': rank_years[6],
     'ranking2011': rank_years[7],
     'ranking2010': rank_years[8],
     'ranking2009': rank_years[9],
     'ranking2008': rank_years[10]}

# create dataframe
df_final = pd.DataFrame(d)
# df_final.to_csv('/Users/glosophy/Dropbox/Human Freedom Index/2020/Data/final.csv', index=False)
df_final.to_csv('final.csv', index=False)

print('csv FILE CREATED! :)')

max_stars_repo_name: glosophy/CatoHFI

max_issues_repo_name: glosophy/CatoHFI

max_forks_repo_name: glosophy/CatoHFI


text:
"""Test cases for evaluation scripts."""
import os
import unittest

import numpy as np

from ..common.typing import NDArrayF64
from ..label.io import load, load_label_config
from ..unittest.util import get_test_file
from .sem_seg import evaluate_sem_seg

class TestScalabelSemSegEval(unittest.TestCase):
    """Test cases for Scalabel semantic segmentation evaluation."""

    cur_dir = os.path.dirname(os.path.abspath(__file__))
    gts_path = f"{cur_dir}/testcases/sem_seg/sem_seg_sample.json"
    preds_path = f"{cur_dir}/testcases/sem_seg/sem_seg_preds.json"
    gts = load(gts_path).frames
    preds = load(preds_path).frames
    config = load_label_config(get_test_file("sem_seg/sem_seg_configs.toml"))
    result = evaluate_sem_seg(gts, preds, config, nproc=1)

    def test_frame(self) -> None:
        """Test case for the function frame()."""
        data_frame = self.result.pd_frame()
        categories = set(
            [
                "road",
                "sidewalk",
                "building",
                "wall",
                "fence",
                "pole",
                "traffic light",
                "traffic sign",
                "vegetation",
                "terrain",
                "sky",
                "person",
                "rider",
                "car",
                "bicycle",
                "bus",
                "motorcycle",
                "train",
                "truck",
                "AVERAGE",
            ]
        )
        self.assertSetEqual(categories, set(data_frame.index.values))

        data_arr = data_frame.to_numpy()
        aps: NDArrayF64 = np.array(
            [
                99.54405124,
                87.67803909,
                95.88390161,
                0.0,
                22.64488433,
                64.17477739,
                0.0,
                55.16129032,
                80.93376685,
                0.0,
                98.23104973,
                0.0,
                0.0,
                94.99133267,
                0.0,
                0.0,
                0.0,
                0.0,
                0.0,
                77.69367703,
            ],
            dtype=np.float64,
        )
        self.assertTrue(
            np.isclose(np.nan_to_num(data_arr[:, 0], nan=-1.0), aps).all()
        )

        overall_scores: NDArrayF64 = np.array(
            [77.69367703, 85.08073851], dtype=np.float64
        )
        self.assertTrue(
            np.isclose(
                np.nan_to_num(data_arr[-1], nan=-1.0), overall_scores
            ).all()
        )

    def test_summary(self) -> None:
        """Check evaluation scores' correctness."""
        summary = self.result.summary()
        overall_reference = {
            "mIoU": 77.69367702513847,
            "mAcc": 85.08073850792114,
            "fIoU": 95.96580841723501,
            "pAcc": 97.59265266143856,
        }
        self.assertSetEqual(set(summary.keys()), set(overall_reference.keys()))
        for name, score in summary.items():
            if np.isnan(score):
                score = np.nan_to_num(score, nan=-1.0)
            self.assertAlmostEqual(score, overall_reference[name])

class TestScalabelSemSegEvalEmpty(unittest.TestCase):
    """Test cases for Scalabel instance segmentation on empty test cases."""

    cur_dir = os.path.dirname(os.path.abspath(__file__))
    gts_path = f"{cur_dir}/testcases/sem_seg/sem_seg_sample.json"
    preds_path = f"{cur_dir}/testcases/sem_seg/sem_seg_preds_empty.json"
    gts = load(gts_path).frames
    preds = load(preds_path).frames
    config = load_label_config(get_test_file("sem_seg/sem_seg_configs.toml"))
    result = evaluate_sem_seg(gts, preds, config, nproc=1)

    def test_frame(self) -> None:
        """Test case for the function frame()."""
        data_frame = self.result.pd_frame()
        categories = set(
            [
                "road",
                "sidewalk",
                "building",
                "fence",
                "wall",
                "pole",
                "traffic light",
                "traffic sign",
                "terrain",
                "vegetation",
                "sky",
                "person",
                "rider",
                "bicycle",
                "bus",
                "car",
                "motorcycle",
                "train",
                "truck",
                "AVERAGE",
            ]
        )
        self.assertSetEqual(categories, set(data_frame.index.values))

        data_arr = data_frame.to_numpy()
        aps: NDArrayF64 = np.array([0.0] * 20, dtype=np.float64)
        self.assertTrue(
            np.isclose(np.nan_to_num(data_arr[:, 0], nan=-1.0), aps).all()
        )

        overall_scores: NDArrayF64 = np.array([0.0] * 2, dtype=np.float64)
        self.assertTrue(
            np.isclose(
                np.nan_to_num(data_arr[-1], nan=-1.0), overall_scores
            ).all()
        )

    def test_summary(self) -> None:
        """Check evaluation scores' correctness."""
        summary = self.result.summary()
        overall_reference = {
            "mIoU": 0.0,
            "mAcc": 0.0,
            "fIoU": 0.0,
            "pAcc": 0.0,
        }
        self.assertSetEqual(set(summary.keys()), set(overall_reference.keys()))
        for name, score in summary.items():
            if np.isnan(score):
                score = np.nan_to_num(score, nan=-1.0)
            self.assertAlmostEqual(score, overall_reference[name])

max_stars_repo_name: Anthuang/scalabel

max_issues_repo_name: Anthuang/scalabel

max_forks_repo_name: Anthuang/scalabel


text:
from sympy import FiniteSet, pi
def time_period(length, g):
    T = 2*pi*(length/g)**0.5
    return T

if __name__ == '__main__':
    L = FiniteSet(15,18,21,22.5,25)
    g_values = FiniteSet(9.8, 9.78, 9.83)
    print('{0:^15}{1:^15}{2:^15}'.format('Length(cm)','Gravity(m/s^2)','Time  Period(s)'))
    for elem in L*g_values:
        l = elem[0]
        g = elem[1]
        t = time_period(l/100, g)

        print('{0:^15}{1:^15}{2:^15.3f}'.format(float(l),float(g),float(t)))

reason: from sympy


text:
#!/usr/bin/python

# MIT License
#
# Copyright (c) 2018-2021 Stefan Chmiela
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import argparse
import hashlib
import os
import re
import sys

import numpy as np

from . import ui

_z_str_to_z_dict = {
    'H': 1,
    'He': 2,
    'Li': 3,
    'Be': 4,
    'B': 5,
    'C': 6,
    'N': 7,
    'O': 8,
    'F': 9,
    'Ne': 10,
    'Na': 11,
    'Mg': 12,
    'Al': 13,
    'Si': 14,
    'P': 15,
    'S': 16,
    'Cl': 17,
    'Ar': 18,
    'K': 19,
    'Ca': 20,
    'Sc': 21,
    'Ti': 22,
    'V': 23,
    'Cr': 24,
    'Mn': 25,
    'Fe': 26,
    'Co': 27,
    'Ni': 28,
    'Cu': 29,
    'Zn': 30,
    'Ga': 31,
    'Ge': 32,
    'As': 33,
    'Se': 34,
    'Br': 35,
    'Kr': 36,
    'Rb': 37,
    'Sr': 38,
    'Y': 39,
    'Zr': 40,
    'Nb': 41,
    'Mo': 42,
    'Tc': 43,
    'Ru': 44,
    'Rh': 45,
    'Pd': 46,
    'Ag': 47,
    'Cd': 48,
    'In': 49,
    'Sn': 50,
    'Sb': 51,
    'Te': 52,
    'I': 53,
    'Xe': 54,
    'Cs': 55,
    'Ba': 56,
    'La': 57,
    'Ce': 58,
    'Pr': 59,
    'Nd': 60,
    'Pm': 61,
    'Sm': 62,
    'Eu': 63,
    'Gd': 64,
    'Tb': 65,
    'Dy': 66,
    'Ho': 67,
    'Er': 68,
    'Tm': 69,
    'Yb': 70,
    'Lu': 71,
    'Hf': 72,
    'Ta': 73,
    'W': 74,
    'Re': 75,
    'Os': 76,
    'Ir': 77,
    'Pt': 78,
    'Au': 79,
    'Hg': 80,
    'Tl': 81,
    'Pb': 82,
    'Bi': 83,
    'Po': 84,
    'At': 85,
    'Rn': 86,
    'Fr': 87,
    'Ra': 88,
    'Ac': 89,
    'Th': 90,
    'Pa': 91,
    'U': 92,
    'Np': 93,
    'Pu': 94,
    'Am': 95,
    'Cm': 96,
    'Bk': 97,
    'Cf': 98,
    'Es': 99,
    'Fm': 100,
    'Md': 101,
    'No': 102,
    'Lr': 103,
    'Rf': 104,
    'Db': 105,
    'Sg': 106,
    'Bh': 107,
    'Hs': 108,
    'Mt': 109,
    'Ds': 110,
    'Rg': 111,
    'Cn': 112,
    'Uuq': 114,
    'Uuh': 116,
}
_z_to_z_str_dict = {v: k for k, v in _z_str_to_z_dict.items()}

def z_str_to_z(z_str):
    return np.array([_z_str_to_z_dict[x] for x in z_str])

def z_to_z_str(z):
    return [_z_to_z_str_dict[int(x)] for x in z]

def train_dir_name(
    dataset, n_train, use_sym, use_cprsn, use_E, use_E_cstr
):

    theory_level_str = re.sub(r'[^\w\-_\.]', '.', str(dataset['theory']))
    theory_level_str = re.sub(r'\.\.', '.', theory_level_str)

    sym_str = '-sym' if use_sym else ''
    cprsn_str = '-cprsn' if use_cprsn else ''
    noE_str = '-noE' if not use_E else ''
    Ecstr_str = '-Ecstr' if use_E_cstr else ''

    return 'sgdml_cv_%s-%s-train%d%s%s%s%s' % (
        dataset['name'].astype(str),
        theory_level_str,
        n_train,
        sym_str,
        cprsn_str,
        noE_str,
        Ecstr_str,
    )

def task_file_name(task):

    n_train = task['idxs_train'].shape[0]
    n_perms = task['perms'].shape[0]
    sig = np.squeeze(task['sig'])

    return 'task-train%d-sym%d-sig%04d.npz' % (n_train, n_perms, sig)

def model_file_name(task_or_model, is_extended=False):

    n_train = task_or_model['idxs_train'].shape[0]
    n_perms = task_or_model['perms'].shape[0]
    sig = np.squeeze(task_or_model['sig'])

    if is_extended:
        dataset = np.squeeze(task_or_model['dataset_name'])
        theory_level_str = re.sub(
            r'[^\w\-_\.]', '.', str(np.squeeze(task_or_model['dataset_theory']))
        )
        theory_level_str = re.sub(r'\.\.', '.', theory_level_str)
        return '%s-%s-train%d-sym%d.npz' % (dataset, theory_level_str, n_train, n_perms)
    return 'model-train%d-sym%d-sig%04d.npz' % (n_train, n_perms, sig)

def dataset_md5(dataset):

    md5_hash = hashlib.md5()

    keys = ['z', 'R']
    if 'E' in dataset:
        keys.append('E')

    keys.append('F')

    if 'perms' in dataset:
        keys.append('perms')
    # only include new extra keys in fingerprint for 'modern' dataset files
    # 'code_version' was included from 0.4.0.dev1
    # opt_keys = ['lattice', 'e_unit', 'E_min', 'E_max', 'E_mean', 'E_var', 'f_unit', 'F_min', 'F_max', 'F_mean', 'F_var']
    # for k in opt_keys:
    #    if k in dataset:
    #        keys.append(k)

    for k in keys:
        d = dataset[k]
        if type(d) is np.ndarray:
            d = d.ravel()
        md5_hash.update(hashlib.md5(d).digest())

    return md5_hash.hexdigest().encode('utf-8')

# ## FILES

# Read geometry file (xyz format).
# R: (n_geo,3*n_atoms)
# z: (3*n_atoms,)
def read_xyz(file_path):

    with open(file_path, 'r') as f:
        n_atoms = None

        R, z = [], []
        for i, line in enumerate(f):
            line = line.strip()
            if not n_atoms:
                n_atoms = int(line)

            cols = line.split()
            file_i, line_i = divmod(i, n_atoms + 2)
            if line_i >= 2:
                R.append(list(map(float, cols[1:4])))
                if file_i == 0:  # first molecule
                    z.append(_z_str_to_z_dict[cols[0]])

        R = np.array(R).reshape(-1, 3 * n_atoms)
        z = np.array(z)

        f.close()
    return R, z

# Write geometry file (xyz format).
def write_geometry(filename, r, z, comment_str=''):

    r = np.squeeze(r)
    try:
        with open(filename, 'w') as f:
            f.write(str(len(r)) + '\n' + comment_str)
            for i, atom in enumerate(r):
                f.write('\n' + _z_to_z_str_dict[z[i]] + '\t')
                f.write('\t'.join(str(x) for x in atom))
    except IOError:
        sys.exit("ERROR: Writing xyz file failed.")

# Write geometry file (xyz format).
def generate_xyz_str(r, z, e=None, f=None, lattice=None):

    comment_str = ''
    if lattice is not None:
        comment_str += 'Lattice=\"{}\" '.format(
            ' '.join(['{:.12g}'.format(l) for l in lattice.T.ravel()])
        )
    if e is not None:
        comment_str += 'Energy={:.12g} '.format(e)
    comment_str += 'Properties=species:S:1:pos:R:3'
    if f is not None:
        comment_str += ':forces:R:3'

    species_str = '\n'.join([_z_to_z_str_dict[z_i] for z_i in z])

    r_f_str = ui.gen_mat_str(r)[0]
    if f is not None:
        r_f_str = ui.merge_col_str(r_f_str, ui.gen_mat_str(f)[0])

    xyz_str = str(len(r)) + '\n' + comment_str + '\n'
    xyz_str += ui.merge_col_str(species_str, r_f_str)

    return xyz_str

def lattice_vec_to_par(lat):

    cell = lat.T
    lengths = [np.linalg.norm(v) for v in cell]

    angles = []
    for i in range(3):
        j = i - 1
        k = i - 2

        ll = lengths[j] * lengths[k]
        if ll > 1e-16:
            x = np.dot(cell[j], cell[k]) / ll
            angle = 180.0 / np.pi * np.arccos(x)
        else:
            angle = 90.0
        angles.append(angle)

    return lengths, angles

### FILE HANDLING

def is_file_type(arg, type):
    """
    Validate file path and check if the file is of the specified type.

    Parameters
    ----------
        arg : :obj:`str`
            File path.
        type : {'dataset', 'task', 'model'}
            Possible file types.

    Returns
    -------
        (:obj:`str`, :obj:`dict`)
            Tuple of file path (as provided) and data stored in the
            file. The returned instance of NpzFile class must be
            closed to avoid leaking file descriptors.

    Raises
    ------
        ArgumentTypeError
            If the provided file path does not lead to a NpzFile.
        ArgumentTypeError
            If the file is not readable.
        ArgumentTypeError
            If the file is of wrong type.
        ArgumentTypeError
            If path/fingerprint is provided, but the path is not valid.
        ArgumentTypeError
            If fingerprint could not be resolved.
        ArgumentTypeError
            If multiple files with the same fingerprint exist.

    """

    # Replace MD5 dataset fingerprint with file name, if necessary.
    if type == 'dataset' and not arg.endswith('.npz') and not os.path.isdir(arg):
        dir = '.'
        if re.search(r'^[a-f0-9]{32}$', arg):  # arg looks similar to MD5 hash string
            md5_str = arg
        else:  # is it a path with a MD5 hash at the end?
            md5_str = os.path.basename(os.path.normpath(arg))
            dir = os.path.dirname(os.path.normpath(arg))

            if dir == '':  # it is only a filename after all, hence not the right type
                raise argparse.ArgumentTypeError('{0} is not a .npz file'.format(arg))

            if re.search(r'^[a-f0-9]{32}$', md5_str) and not os.path.isdir(
                dir
            ):  # path has MD5 hash string at the end, but directory is not valid
                raise argparse.ArgumentTypeError('{0} is not a directory'.format(dir))

        file_names = filter_file_type(dir, type, md5_match=md5_str)

        if not len(file_names):
            raise argparse.ArgumentTypeError(
                "No {0} files with fingerprint '{1}' found in '{2}'".format(
                    type, md5_str, dir
                )
            )
        elif len(file_names) > 1:
            error_str = (
                "Multiple {0} files with fingerprint '{1}' found in '{2}'".format(
                    type, md5_str, dir
                )
            )
            for file_name in file_names:
                error_str += '\n       {0}'.format(file_name)

            raise argparse.ArgumentTypeError(error_str)
        else:
            arg = os.path.join(dir, file_names[0])

    if not arg.endswith('.npz'):
        argparse.ArgumentTypeError('{0} is not a .npz file'.format(arg))

    try:
        file = np.load(arg, allow_pickle=True)
    except Exception:
        raise argparse.ArgumentTypeError('{0} is not readable'.format(arg))

    if 'type' not in file or file['type'].astype(str) != type[0]:
        raise argparse.ArgumentTypeError('{0} is not a {1} file'.format(arg, type))

    return arg, file

def filter_file_type(dir, type, md5_match=None):
    """
    Filters all files from a directory that match a given type and (optionally)
    a given fingerprint.

    Parameters
    ----------
        arg : :obj:`str`
            File path.
        type : {'dataset', 'task', 'model'}
            Possible file types.
        md5_match : :obj:`str`, optional
            Fingerprint string.

    Returns
    -------
        :obj:`list` of :obj:`str`
            List of file names that match the specified type and fingerprint
            (if provided).

    Raises
    ------
        ArgumentTypeError
            If the directory contains unreadable .npz files.

    """

    file_names = []
    for file_name in sorted(os.listdir(dir)):
        if file_name.endswith('.npz'):
            file_path = os.path.join(dir, file_name)
            try:
                file = np.load(file_path, allow_pickle=True)
            except Exception:
                raise argparse.ArgumentTypeError(
                    '{0} contains unreadable .npz files'.format(arg)
                )

            if 'type' in file and file['type'].astype(str) == type[0]:

                if md5_match is None:
                    file_names.append(file_name)
                elif 'md5' in file and file['md5'] == md5_match:
                    file_names.append(file_name)

            file.close()

    return file_names

def is_valid_file_type(arg_in):
    """
    Check if file is either a valid dataset, task or model file.

    Parameters
    ----------
        arg_in : :obj:`str`
            File path.

    Returns
    -------
        (:obj:`str`, :obj:`dict`)
            Tuple of file path (as provided) and data stored in the
            file. The returned instance of NpzFile class must be
            closed to avoid leaking file descriptors.

    Raises
    ------
        ArgumentTypeError
            If the provided file path does not point to a supported
            file type.

    """

    arg, file = None, None
    try:
        arg, file = is_file_type(arg_in, 'dataset')
    except argparse.ArgumentTypeError:
        pass

    if file is None:
        try:
            arg, file = is_file_type(arg_in, 'task')
        except argparse.ArgumentTypeError:
            pass

    if file is None:
        try:
            arg, file = is_file_type(arg_in, 'model')
        except argparse.ArgumentTypeError:
            pass

    if file is None:
        raise argparse.ArgumentTypeError(
            '{0} is neither a dataset, task, nor model file'.format(arg)
        )

    return arg, file

def is_dir_with_file_type(arg, type, or_file=False):
    """
    Validate directory path and check if it contains files of the specified type.

    Note
    ----
        If a file path is provided, this function acts like its a directory with
        just one file.

    Parameters
    ----------
        arg : :obj:`str`
            File path.
        type : {'dataset', 'task', 'model'}
            Possible file types.
        or_file : bool
            If `arg` contains a file path, act like it's a directory
            with just a single file inside.

    Returns
    -------
        (:obj:`str`, :obj:`list` of :obj:`str`)
            Tuple of directory path (as provided) and a list of
            contained file names of the specified type.

    Raises
    ------
        ArgumentTypeError
            If the provided directory path does not lead to a directory.
        ArgumentTypeError
            If directory contains unreadable files.
        ArgumentTypeError
            If directory contains no files of the specified type.
    """

    if or_file and os.path.isfile(arg):  # arg: file path
        _, file = is_file_type(
            arg, type
        )  # raises exception if there is a problem with the file
        file.close()
        file_name = os.path.basename(arg)
        file_dir = os.path.dirname(arg)
        return file_dir, [file_name]
    else:  # arg: dir

        if not os.path.isdir(arg):
            raise argparse.ArgumentTypeError('{0} is not a directory'.format(arg))

        file_names = filter_file_type(arg, type)

        if not len(file_names):
            raise argparse.ArgumentTypeError(
                '{0} contains no {1} files'.format(arg, type)
            )

        return arg, file_names

def is_task_dir_resumeable(
    train_dir, train_dataset, test_dataset, n_train, n_test, sigs, gdml
):
    r"""
    Check if a directory contains `task` and/or `model` files that
    match the configuration of a training process specified in the
    remaining arguments.

    Check if the training and test datasets in each task match
    `train_dataset` and `test_dataset`, if the number of training and
    test points matches and if the choices for the kernel
    hyper-parameter :math:`\sigma` are contained in the list. Check
    also, if the existing tasks/models contain symmetries and if
    that's consistent with the flag `gdml`. This function is useful
    for determining if a training process can be resumed using the
    existing files or not.

    Parameters
    ----------
        train_dir : :obj:`str`
            Path to training directory.
        train_dataset : :obj:`dataset`
            Dataset from which training points are sampled.
        test_dataset : :obj:`test_dataset`
            Dataset from which test points are sampled (may be the
            same as `train_dataset`).
        n_train : int
            Number of training points to sample.
        n_test : int
            Number of test points to sample.
        sigs : :obj:`list` of int
            List of :math:`\sigma` kernel hyper-parameter choices
            (usually: the hyper-parameter search grid)
        gdml : bool
            If `True`, don't include any symmetries in model (GDML),
            otherwise do (sGDML).

    Returns
    -------
        bool
            False, if any of the files in the directory do not match
            the training configuration.
    """

    for file_name in sorted(os.listdir(train_dir)):
        if file_name.endswith('.npz'):
            file_path = os.path.join(train_dir, file_name)
            file = np.load(file_path, allow_pickle=True)

            if 'type' not in file:
                continue
            elif file['type'] == 't' or file['type'] == 'm':

                if (
                    file['md5_train'] != train_dataset['md5']
                    or file['md5_valid'] != test_dataset['md5']
                    or len(file['idxs_train']) != n_train
                    or len(file['idxs_valid']) != n_test
                    or gdml
                    and file['perms'].shape[0] > 1
                    or file['sig'] not in sigs
                ):
                    return False

    return True

### ARGUMENT VALIDATION

def is_strict_pos_int(arg):
    """
    Validate strictly positive integer input.

    Parameters
    ----------
        arg : :obj:`str`
            Integer as string.

    Returns
    -------
        int
            Parsed integer.

    Raises
    ------
        ArgumentTypeError
            If integer is not > 0.
    """
    x = int(arg)
    if x <= 0:
        raise argparse.ArgumentTypeError('must be strictly positive')
    return x

def parse_list_or_range(arg):
    """
    Parses a string that represents either an integer or a range in
    the notation ``<start>:<step>:<stop>``.

    Parameters
    ----------
        arg : :obj:`str`
            Integer or range string.

    Returns
    -------
        int or :obj:`list` of int

    Raises
    ------
        ArgumentTypeError
            If input can neither be interpreted as an integer nor a valid range.
    """

    if re.match(r'^\d+:\d+:\d+$', arg) or re.match(r'^\d+:\d+$', arg):
        rng_params = list(map(int, arg.split(':')))

        step = 1
        if len(rng_params) == 2:  # start, stop
            start, stop = rng_params
        else:  # start, step, stop
            start, step, stop = rng_params

        rng = list(range(start, stop + 1, step))  # include last stop-element in range
        if len(rng) == 0:
            raise argparse.ArgumentTypeError('{0} is an empty range'.format(arg))

        return rng
    elif re.match(r'^\d+$', arg):
        return int(arg)

    raise argparse.ArgumentTypeError(
        '{0} is neither a integer list, nor valid range in the form <start>:[<step>:]<stop>'.format(
            arg
        )
    )

max_stars_repo_name: AntonCh-G/sGDML

max_issues_repo_name: AntonCh-G/sGDML

max_forks_repo_name: AntonCh-G/sGDML


text:
import numpy as np
import os

from datasets_utils.morphing_space import transform_morph_space_list2space
from utils.load_config import load_config
from utils.load_data import load_data
from models.NormBase import NormBase
from plots_utils.plot_morphing_space import plot_it_resp
from plots_utils.plot_morphing_space import plot_morphing_space

"""
test the naive PCA selection on the original morphing space dataset from the eLife paper

run: python -m projects.facial_shape_expression_recognition_transfer.01b_NB_morphing_space
"""

train_model = False
predict_it_resp = False

# load config
config_name = 'NB_morphing_space_m0001.json'
config = load_config(config_name, path='configs/norm_base_config')

# --------------------------------------------------------------------------------------------------------------------
# declare model
model = NormBase(config, input_shape=tuple(config['input_shape']), load_NB_model=True)

# train model
if train_model:
    print("[FIT] Train model]")
    data = load_data(config, train=True)
    model.fit(data,
              fit_dim_red=True,
              fit_ref=True,
              fit_tun=True)
    model.save()

# --------------------------------------------------------------------------------------------------------------------
# predict model
if predict_it_resp:
    data = load_data(config, train=False)
    print("[Test] Predict model")
    accuracy, it_resp, labels = model.evaluate(data)
    print("[Test] model predicted")
    np.save(os.path.join("models/saved", config['config_name'], "NormBase", "it_resp"), it_resp)
    print("[Test] it_resp saved!")
else:
    it_resp = np.load(os.path.join("models/saved", config['config_name'], "NormBase", "it_resp.npy"))
    print("[Test] it_resp loaded!")
print("[Test] shape it_resp", np.shape(it_resp))
print()

# --------------------------------------------------------------------------------------------------------------------
# plot

# reshape responses to a 25 stimuli x 150 frames x n_category
it_resp = np.reshape(it_resp, (25, 150, np.shape(it_resp)[-1]))
# remove the neutral responses
it_resp = it_resp[..., 1:]
print("[Plot] shape it_resp", np.shape(it_resp))

# plot it responses of the training prototypes
it_proto = it_resp[[20, 0, 24, 4]]  # keep only the 4 prototypes c1=20, c2=0, c3=24, c4=4
print("[Plot] shape it_proto", np.shape(it_proto))

plot_it_resp(it_proto, title="NB Human Prototypes",
             labels=config['train_expression'],
             save_folder=os.path.join(os.path.join("models/saved", config['config_name'])))

# plot the responses over the entire morphing space
# reshape the number of stimuli to a 5x5 grid
it_morph_space = transform_morph_space_list2space(it_resp)
print("[Plot] shape morphing space", np.shape(it_morph_space))
# it_morph_space = np.amax(it_morph_space, axis=2)  # compute max over all sequences
it_morph_space = np.sum(it_morph_space, axis=2)  # compute max over all sequences
print("[Plot] shape it_morph_space", np.shape(it_morph_space))
plot_morphing_space(it_morph_space,
                    save_folder=os.path.join(os.path.join("models/saved", config['config_name'])))

max_stars_repo_name: michaelStettler/BVS

max_issues_repo_name: michaelStettler/BVS

max_forks_repo_name: michaelStettler/BVS


text:
from numpy.testing._private.utils import tempdir
import tvm
from tvm import relay
from tvm.runtime.ndarray import *
import math
import struct
from tvm import relay, auto_scheduler
from tvm import IRModule
import numpy as np
import binascii
from tvm.autotvm.database import Database
from tvm.driver.build_module import lower
from tvm.contrib import graph_runtime , graph_executor
import numpy as np

network='testmd5'
target=tvm.target.Target('llvm')
dtype='int32'
log_file="%s-%s.json"%(network,target.kind.name)

# 输入数据
# A_data = 0b00000001
# print('A_data:',A_data)
B_data = 0b00010101
print('B_data:',B_data)
C_data = 0b01000001
print('C_data:',C_data)
D_data = 0b01010100
print('D_data:',D_data)
print('  ')

# 测试数据
# f=(b&c)|((~b)&d)
notB=~ B_data
print("notB:",notB)
BandC=B_data & C_data
print("BandC:",BandC)
notBandD=(notB & D_data)
print("notBAndD:",notBandD)
res=BandC | notBandD
print("res:",res)
print("    ")

#输入参数
B_data = np.array([[B_data]],);
C_data = np.array([[C_data]]);
D_data = np.array([[D_data]]);
params = {"B":B_data,"C":C_data,"D":D_data}
# params = {"B":B_data,"C":C_data,"D":D_data}
# params={}
# 输入变量

B = relay.var('B', shape=[1, 1], dtype='int32')
C = relay.var('C', shape=[1, 1], dtype='int32')
D = relay.var('D', shape=[1, 1], dtype='int32')

# f = (b & c) | ((~b) & d)
notB = relay.bitwise_not(B)
bAndC = relay.bitwise_and(B, C)
notBAndD = relay.bitwise_and(notB, D)
f = relay.bitwise_or(bAndC, notBAndD)

# 将多个字段组合在一起的元组表达式
func=relay.Tuple((f,bAndC,notBAndD,notB))
print("func: ",func)
fun=relay.Function([B,C,D],func)
print("  ")
print("fun： ",fun)
print("  ")

# 生成模型表达式
mod=IRModule.from_expr(fun)

# relay.build返回三个组建，json格式的编译后的最终图graph,lib可以使用TVM运行时在目标上部署的目标库。,params编译后的最终参数。
with tvm.transform.PassContext(opt_level=3):
    # graph, lib, params = relay.build(mod, "llvm", params=params)
    lib= relay.build(mod, "llvm", params=None)
    m = tvm.contrib.graph_executor.GraphModule(lib["default"](tvm.cpu()))
    #print("graph1:",graph)
    print("lib: ",lib)
# lib.export_library("compiled_lib_test.so")

# 输出结果
#mod=graph_executor.create(graph,lib,device=tvm.cpu(0))
m.set_input(**params)
# mod.set_input(B=B_data,C=C_data,D=D_data)
m.run()
result=m.get_output(0).asnumpy();
print("result: ",result)

max_stars_repo_name: new-TonyWang/tvm

max_issues_repo_name: new-TonyWang/tvm

max_forks_repo_name: new-TonyWang/tvm

reason: import numpy,from numpy


text:
# import MDAnalysis as md
# from openmm import Vec3
# from openmm.unit import nanometer
import mdtraj as md
import os
from sys import stdout
import numpy as np
from pathlib import Path
import sys

"""
def get_openmm_pos_from_traj(top, ref_traj, modif_traj, selected_atoms='protein', write_dcd=True,
                             write_pdb_for_selection=True, start=None, stop=None):
    global first_snapshot_name
    positions = []
    All_Positions = {}

    save_directory = os.path.dirname(ref_traj)

    # set the universe object
    u = md.Universe(top, ref_traj, modif_traj)

    # set atom selection for all atoms of interest
    wanted_atoms = u.select_atoms(selected_atoms)

    # Writer object that spans the whole trajectory and
    # writes an abbreviated trajectory only including atoms described by atom selection
    for cnt in u.trajectory:
        positions.append(wanted_atoms.positions)

    for k, i in enumerate(positions):
        Pos = []
        for xyzi in i:
            Pos.append(Vec3(xyzi[0], xyzi[1], xyzi[2]))

        All_Positions[k] = (Pos * nanometer)
        del Pos

    if write_dcd:
        if start and stop is not None:
            with md.Writer(os.path.join(save_directory, '%s_%s_%s.dcd' % (ref_traj, start, stop)),
                                   len(wanted_atoms.atoms)) as V:
                for ts in u.trajectory[start:stop]:
                    V.write(wanted_atoms)

        if start is None and stop is not None:
            start = 0
            with md.Writer(os.path.join(save_directory, '%s_%s_%s.dcd' % (ref_traj, start, stop)),
                                   len(wanted_atoms.atoms)) as V:
                for ts in u.trajectory[start:stop]:
                    V.write(wanted_atoms)

        if start is not None and stop is None:
            with md.Writer(os.path.join(save_directory, '%s_%s_all.dcd' % (ref_traj, start)),
                                   len(wanted_atoms.atoms)) as V:
                for ts in u.trajectory[start:]:
                    V.write(wanted_atoms)

    if write_pdb_for_selection:
        first_snapshot_name = os.path.join(save_directory, '%s_first_snapshot.pdb' % selected_atoms)
        wanted_atoms.write(first_snapshot_name)

    return All_Positions, first_snapshot_name
"""

def create_restrained_atoms_topology(pdb_file):
    print("Creating topology with only the restrained atoms.")

    file = md.load(pdb_file).remove_solvent()
    topology = file.topology

    box_dimension = file.openmm_boxes(0)

    all_atoms_indices = []

    for i in range(topology.n_residues):
        all_atoms_indices.append(topology.select("resid %s" % i))

    return all_atoms_indices, topology.to_openmm(), box_dimension

def get_openmm_pos_from_traj_with_mdtraj(top, ref_traj, modif_traj, selected_atoms='protein',
                                         write_pdb_for_selection=True, start=None, stop=None, stride=1):
    global first_snapshot_name
    positions = []
    save_directory = os.path.dirname(modif_traj)

    trajectory_collector = object
    traj1 = object
    traj2 = object
    if ref_traj is not None:
        traj1 = md.join(md.iterload(ref_traj, chunk=100, stride=stride, atom_indices=None, top=top))
        traj2 = md.join(md.iterload(modif_traj, chunk=100, stride=stride, atom_indices=None, top=top))
        trajectory_collector = traj1 + traj2

    if ref_traj is None:
        trajectory_collector = md.join(md.iterload(modif_traj, chunk=100, stride=stride, atom_indices=None, top=top))

    # set atom selection for all atoms of interest
    wanted_atoms_traj = trajectory_collector.atom_slice(trajectory_collector.topology.select(selected_atoms))

    print("Started to convert Traj positions for OpenMM..")
    for i in range(wanted_atoms_traj.n_frames):
        positions.append(wanted_atoms_traj.openmm_positions(i))

    if write_pdb_for_selection:
        first_snapshot_name = os.path.join(save_directory, '%s_first_snapshot.pdb' % selected_atoms)
        wanted_atoms_traj[0].save_pdb(first_snapshot_name)

    del traj1, traj2, trajectory_collector
    return positions, first_snapshot_name

# traj = 'trajectory.dcd'
# pdb_file = 'last_structure.pdb'

# all_atoms_indices, new_topology, box_vectors = create_restrained_atoms_topology(pdb_file)
#
# # # print(new_topology)
# All_positions, pdb = get_openmm_pos_from_traj(pdb_file, traj, write_dcd=False)

#
# file = app.PDBFile(pdb)
# top = file.topology
# pos = file.positions
#
#
#
#
#
# print('Forcefield parameters loading to the simulation system ...')
# forcefield = app.ForceField('charmm36.xml')
# #
# #
# #
# #
# print('Constructing an OpenMM System')
# system = forcefield.createSystem(top, nonbondedMethod=app.PME,
#                                  nonbondedCutoff=12.0 * unit.angstroms, constraints=None,
#                                  rigidWater=False,
#                                  ewaldErrorTolerance=5e-5)
#

# print(table)
# topology = topology.subset(self._reporter.analysis_particle_indices)
# # Use the receptor as an anchor molecule and image the ligand.
# anchor_molecules = [{a for a in topology.atoms if a.index in set(topography.receptor_atoms)}]
# imaged_molecules = [{a for a in topology.atoms if a.index in set(topography.ligand_atoms)}]
# # Initialize trajectory object needed for imaging molecules.
# trajectory = mdtraj.Trajectory(xyz=np.zeros((topology.n_atoms, 3)), topology=topology)


text:
from orangecontrib.recommendation.ranking import Learner, Model
from orangecontrib.recommendation.utils.format_data import *
from orangecontrib.recommendation.optimizers import *
from orangecontrib.recommendation.utils.datacaching import cache_rows

from collections import defaultdict

from scipy.special import expit as sigmoid

import numpy as np
import time
import warnings

__all__ = ['CLiMFLearner']
__sparse_format__ = lil_matrix

def _g(x):
    """sigmoid function"""
    return sigmoid(x)

def _dg(x):
    ex = np.exp(-x)
    y = ex / (1 + ex) ** 2
    return y

def _matrix_factorization(ratings, shape, num_factors, num_iter, learning_rate,
                          lmbda, optimizer, verbose=False, random_state=None,
                          callback=None):
    # Seed the generator
    if random_state is not None:
        np.random.seed(random_state)

    # Get featured matrices dimensions
    num_users, num_items = shape

    # Initialize low-rank matrices
    U = 0.01 * np.random.rand(num_users, num_factors)  # User-feature matrix
    V = 0.01 * np.random.rand(num_items, num_factors)  # Item-feature matrix

    # Configure optimizer
    update_ui = create_opt(optimizer, learning_rate).update
    update_vw = create_opt(optimizer, learning_rate).update

    # Cache rows
    users_cached = defaultdict(list)

    # Print information about the verbosity level
    if verbose:
        print('CLiMF factorization started.')
        print('\tLevel of verbosity: ' + str(int(verbose)))
        print('\t\t- Verbosity = 1\t->\t[time/iter]')
        print('\t\t- Verbosity = 2\t->\t[time/iter, loss]')
        print('\t\t- Verbosity = 3\t->\t[time/iter, loss, MRR]')
        print('')

        # Prepare sample of users
        if verbose > 2:
            queries = None
            num_samples = min(num_users, 1000)  # max. number to sample
            users_sampled = np.random.choice(np.arange(num_users), num_samples)

    # Catch warnings
    with warnings.catch_warnings():

        # Turn matching warnings into exceptions
        warnings.filterwarnings('error')
        try:

            # Factorize matrix using SGD
            for step in range(num_iter):
                if verbose:
                    start = time.time()
                    print('- Step: %d' % (step + 1))

                # Send information about the process
                if callback:
                    callback(step + 1)

                # Optimize rating prediction
                for i in range(len(U)):
                    dU = -lmbda * U[i]

                    # Precompute f (f[j] = <U[i], V[j]>)
                    items = cache_rows(ratings, i, users_cached)
                    f = np.einsum('j,ij->i', U[i], V[items])

                    for j in range(len(items)):  # j=items
                        w = items[j]

                        dV = _g(-f[j]) - lmbda * V[w]

                        # For I
                        vec1 = _dg(f[j] - f) * \
                               (1 / (1 - _g(f - f[j])) - 1 / (1 - _g(f[j] - f)))
                        dV += np.einsum('i,j->ij', vec1, U[i]).sum(axis=0)

                        update_vw(-dV, V, w)

                        dU += _g(-f[j]) * V[w]

                        # For II
                        vec2 = (V[items[j]] - V[items])
                        vec3 = _dg(f - f[j]) / (1 - _g(f - f[j]))
                        dU += np.einsum('ij,i->ij', vec2, vec3).sum(axis=0)

                    update_ui(-dU, U, i)

                # Print process
                if verbose:
                    print('\t- Time: %.3fs' % (time.time() - start))

                    if verbose > 1:
                        # Set parameters and compute loss
                        low_rank_matrices = (U, V)
                        params = lmbda
                        objective = compute_loss(ratings, low_rank_matrices, params)
                        print('\t- Training loss: %.3f' % objective)

                        if verbose > 2:
                            model = CLiMFModel(U=U, V=V)
                            mrr, queries = \
                                model.compute_mrr(ratings, users_sampled, queries)
                            print('\t- Train MRR: %.4f' % mrr)
                    print('')

        except RuntimeWarning:
            callback(num_iter) if callback else None
            raise RuntimeError('Training diverged and returned NaN.')

    return U, V

def compute_loss(data, low_rank_matrices, params):

    # Set parameters
    ratings = data
    U, V = low_rank_matrices
    lmbda = params

    # Check data type
    if isinstance(ratings, __sparse_format__):
        pass
    elif isinstance(ratings, Table):
        # Preprocess Orange.data.Table and transform it to sparse
        ratings, order, shape = preprocess(ratings)
        ratings = table2sparse(ratings, shape, order, m_type=__sparse_format__)
    else:
        raise TypeError('Invalid data type')

    # Cache rows
    users_cached = defaultdict(list)

    F = -0.5*lmbda*(np.sum(U*U)+np.sum(V*V))

    for i in range(len(U)):
        # Precompute f (f[j] = <U[i], V[j]>)
        items = cache_rows(ratings, i, users_cached)
        f = np.einsum('j,ij->i', U[i], V[items])

        for j in range(len(items)):  # j=items
            F += np.log(_g(f[j]))
            F += np.log(1 - _g(f - f[j])).sum(axis=0)  # For I
    return F

class CLiMFLearner(Learner):
    """CLiMF: Collaborative Less-is-More Filtering Matrix Factorization

    This model uses stochastic gradient descent to find two low-rank
    matrices: user-feature matrix and item-feature matrix.

    CLiMF is a matrix factorization for scenarios with binary relevance data
    when only a few (k) items are recommended to individual users. It improves top-k
    recommendations through ranking by directly maximizing the Mean Reciprocal
    Rank (MRR).

    Attributes:
        num_factors: int, optional
            The number of latent factors.

        num_iter: int, optional
            The number of passes over the training data (aka epochs).

        learning_rate: float, optional
            The learning rate controlling the size of update steps (general).

        lmbda: float, optional
            Controls the importance of the regularization term (general).
            Avoids overfitting by penalizing the magnitudes of the parameters.

        optimizer: Optimizer, optional
            Set the optimizer for SGD. If None (default), classical SGD will be
            applied.

        verbose: boolean or int, optional
            Prints information about the process according to the verbosity
            level. Values: False (verbose=0), True (verbose=1) and INTEGER

        random_state: int, optional
            Set the seed for the numpy random generator, so it makes the random
            numbers predictable. This a debbuging feature.

        callback: callable

    """

    name = 'CLiMF'

    def __init__(self, num_factors=5, num_iter=25, learning_rate=0.0001,
                 lmbda=0.001, preprocessors=None, optimizer=None, verbose=False,
                 random_state=None, callback=None):
        self.num_factors = num_factors
        self.num_iter = num_iter
        self.learning_rate = learning_rate
        self.lmbda = lmbda
        self.optimizer = SGD() if optimizer is None else optimizer
        self.random_state = random_state
        self.callback = callback

        super().__init__(preprocessors=preprocessors, verbose=verbose)

    def fit_storage(self, data):
        """Fit the model according to the given training data.

        Args:
            data: Orange.data.Table

        Returns:
            self: object
                Returns self.

        """

        # Prepare data
        data = super().prepare_fit(data)

        # Check convergence
        if self.learning_rate == 0:
            warnings.warn("With learning_rate=0, this algorithm does not "
                          "converge well.", stacklevel=2)

        # Transform ratings matrix into a sparse matrix
        data = table2sparse(data, self.shape, self.order,
                            m_type=__sparse_format__)

        # Factorize matrix
        U, V = _matrix_factorization(ratings=data, shape=self.shape,
                                     num_factors=self.num_factors,
                                     num_iter=self.num_iter,
                                     learning_rate=self.learning_rate,
                                     lmbda=self.lmbda, optimizer=self.optimizer,
                                     verbose=self.verbose,
                                     random_state=self.random_state,
                                     callback=self.callback)

        # Construct model
        model = CLiMFModel(U=U, V=V)
        return super().prepare_model(model)

class CLiMFModel(Model):

    def __init__(self, U, V):
        self.U = U
        self.V = V
        super().__init__()

    def predict(self, X, top_k=None):
        """Perform predictions on samples in X for all items.

        Args:
            X: array, optional
                Array with the indices of the users to which make the
                predictions. If None (default), predicts for all users.

            top_k: int, optional
                Returns the k-first predictions. (Do not confuse with
                'top-best').

        Returns:
            C: ndarray, shape = (n_samples, n_items)
                Returns predicted values. A matrix (U, I) with the indices of
                the items recommended, sorted by ascending ranking. (1st better
                than 2nd, than 3rd,...)

        """

        # Check if is an array
        if isinstance(X, np.ndarray):
            if X.ndim != 1:
                X = X[:, self.order[0]]

        elif not isinstance(X, int):  # Check if is not an int
            raise TypeError("'Invalid input data. Array or 'int'")

        # Compute scores
        predictions = np.dot(self.U[X], self.V.T)

        # Return indices of the sorted predictions
        # (Return top-k recommendations, optional)
        if isinstance(X, int):  # Array of 1D
            predictions = np.argsort(predictions)
            predictions = predictions[::-1]
            predictions = predictions[:top_k] if top_k else predictions
        else:
            predictions = np.argsort(predictions, axis=1)
            predictions = np.fliplr(predictions)
            predictions = predictions[:, :top_k] if top_k else predictions

        return predictions

    def getUTable(self):
        variable = self.original_domain.variables[self.order[0]]
        return feature_matrix(variable, self.U)

    def getVTable(self):
        variable = self.original_domain.variables[self.order[1]]
        return feature_matrix(variable, self.V)

reason: import numpy,from scipy