text:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import operator
import collections
import math
import time
import os
import random
import zipfile
import time
import numpy as np
import sys
from six.moves import urllib
from six.moves import xrange  # pylint: disable=redefined-builtin
import tensorflow as tf
sys.path.append( '../util/')
from generators import *
from loader import *
from print_tweets import *
from similar_tokens import *
from training import *
from similar_tokens import *
from expand_query import *
from argument_loader import *
from setup import *
from LSTM import *

# load arguements
dataset, query_type, filename, num_steps, num_steps_roll, num_steps_train, expand_flag,lr_, matchname = import_arguments(sys.argv)
# load data
char_batch_dict, word_batch_dict,data, count, dictionary, reverse_dictionary, word_max_len, char_max_len, vocabulary_size, char_dictionary, reverse_char_dictionary, data_index, char_data_index, buffer_index, batch_list, char_batch_list, word_batch_list, char_data = build_everything("complete", consider=True, consider_set="nepal")
# test the data
data_index, batch, labels = generate_batch(data, data_index, batch_size=8, num_skips=2, skip_window=1,)
for i in range(8):
  print(batch[i], reverse_dictionary[batch[i]],
        '->', labels[i, 0], reverse_dictionary[labels[i, 0]])
char_data_index, batch, labels = generate_batch_char(char_data, char_data_index, batch_size=8, num_skips=2, skip_window=1)
for i in range(8):
  print(batch[i], reverse_char_dictionary[batch[i]],
        '->', labels[i, 0], reverse_char_dictionary[labels[i, 0]])

# load batch values
lambda_1, tweet_batch_size, expand_start_count, query_name, query_tokens, query_tokens_alternate, char_batch_size, num_sampled, valid_examples, valid_window, valid_size, skip_window, num_skips, embedding_size, char_vocabulary_size, batch_size, num_char_skips, skip_char_window = setup(char_dictionary, dictionary, query_type)
learning_rate = lr_
graph = tf.Graph()

with graph.as_default():

  # Input data.
  need_constant = tf.constant(query_tokens,dtype=tf.int32)
  avail_constant = tf.constant(query_tokens_alternate, dtype=tf.int32)
  train_inputs = tf.placeholder(tf.int32, shape=[batch_size])
  train_input_chars = tf.placeholder(tf.int32, shape=[char_batch_size])
  train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
  train_char_labels = tf.placeholder(tf.int32, shape=[char_batch_size, 1])
  valid_dataset = tf.constant(valid_examples[0], dtype=tf.int32)
  word_char_embeddings = tf.placeholder(tf.int32, shape=[batch_size, char_max_len])
  valid_char_dataset = tf.constant(valid_examples[1], dtype=tf.int32)
  expanded_query_ints = tf.placeholder(tf.int32, shape=(len(query_tokens)+3))
  query_ints = tf.placeholder(tf.int32, shape=len(query_tokens))
  tquery_word_holder = tf.placeholder(tf.int32, shape=[word_max_len],name="tweet_query_word_holder")
  tquery_char_holder = tf.placeholder(tf.int32, shape=[word_max_len, char_max_len],name="tweet_query_char_holder")
  # Ops and variables pinned to the CPU because of missing GPU implementation
  tweet_char_holder = tf.placeholder(tf.int32, shape=[tweet_batch_size,word_max_len,char_max_len],name="tweet_char_holder")
  tweet_word_holder = tf.placeholder(tf.int32, shape=[tweet_batch_size, word_max_len],name="tweet_word_holder")
  with tf.device('/gpu:0'):
    # Look up embeddings for inputs.
    embeddings = tf.Variable(tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
    char_embeddings = tf.Variable(tf.random_uniform([char_vocabulary_size, embedding_size],-1.0,1.0))
    embed = tf.nn.embedding_lookup(embeddings, train_inputs)
    char_embed = tf.nn.embedding_lookup(char_embeddings,train_input_chars)

    # Construct the variables for the NCE loss
    nce_weights = tf.Variable(
        tf.truncated_normal([vocabulary_size, embedding_size],
                            stddev=1.0 / math.sqrt(embedding_size)))
    nce_biases = tf.Variable(tf.zeros([vocabulary_size]))
    # character weights
    nce_char_weights = tf.Variable(
        tf.truncated_normal([vocabulary_size, embedding_size],
                            stddev=1.0 / math.sqrt(embedding_size)))
    nce_char_biases = tf.Variable(tf.zeros([vocabulary_size]))

  loss = tf.reduce_mean(
      tf.nn.nce_loss(weights=nce_weights,
                     biases=nce_biases,
                     labels=train_labels,
                     inputs=embed,
                     num_sampled=num_sampled,
                     num_classes=vocabulary_size))

  loss_char = tf.reduce_mean(
      tf.nn.nce_loss(weights=nce_char_weights,
                     biases=nce_char_biases,
                     labels=train_char_labels,
                     inputs=char_embed,
                     num_sampled=10,
                     num_classes=char_vocabulary_size))

  # Construct the SGD optimizer using a learning rate of 1.0.
  optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)
  optimizer_char = tf.train.AdamOptimizer(learning_rate /5).minimize(loss_char)

  # Compute the cosine similarity between minibatch examples and all embeddings.
  norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
  normalized_embeddings = embeddings / norm
  valid_embeddings = tf.nn.embedding_lookup(
      normalized_embeddings, valid_dataset)
  similarity = tf.matmul(
      valid_embeddings, normalized_embeddings, transpose_b=True)
  query_embedding_token = tf.reshape(tf.reduce_mean(tf.nn.embedding_lookup(normalized_embeddings,query_ints),axis=0),shape=[1,embedding_size])
  expanded_query_embedding_token = tf.reshape(tf.reduce_mean(tf.nn.embedding_lookup(normalized_embeddings,expanded_query_ints),axis=0),shape=[1,embedding_size])
  similarity_query = tf.reshape(tf.matmul(
      query_embedding_token, normalized_embeddings, transpose_b=True),shape=[int(normalized_embeddings.shape[0])])
  similarity_expanded_query = tf.reshape(tf.matmul(
      expanded_query_embedding_token, normalized_embeddings, transpose_b=True),shape=[int(normalized_embeddings.shape[0])])

  norm_char = tf.sqrt(tf.reduce_sum(tf.square(char_embeddings), 1, keep_dims=True))
  normalized_char_embeddings = char_embeddings / norm_char
  valid_embeddings_char = tf.nn.embedding_lookup(
      normalized_char_embeddings, valid_char_dataset)
  similarity_char = tf.matmul(
      valid_embeddings_char, normalized_char_embeddings, transpose_b=True)

  tweet_word_embed = tf.nn.embedding_lookup(normalized_embeddings, tweet_word_holder)
  tweet_char_embed = tf.reduce_mean(tf.nn.embedding_lookup(normalized_char_embeddings, tweet_char_holder),axis=2)
  tweet_embedding = tf.reduce_mean(lambda_1*tweet_word_embed + (1-lambda_1)*tweet_char_embed,axis=1)
  query_embedding = tf.reshape(tf.reduce_mean(tf.nn.embedding_lookup(normalized_embeddings,query_ints),axis=0),shape=[1,embedding_size])
  expanded_query_embedding = tf.reshape(tf.reduce_mean(tf.nn.embedding_lookup(normalized_embeddings,expanded_query_ints),axis=0),shape=[1,embedding_size])
  query_similarity = tf.reshape(tf.matmul(tweet_embedding, query_embedding, transpose_b=True),shape=[tweet_batch_size])
  expanded_query_similarity = tf.reshape(tf.matmul(tweet_embedding, expanded_query_embedding, transpose_b=True),shape=[tweet_batch_size])

  tweet_query_char = tf.reduce_mean(tf.nn.embedding_lookup(normalized_char_embeddings, tquery_char_holder),axis=1)
  tweet_query_word = tf.nn.embedding_lookup(normalized_embeddings, tquery_word_holder)
  tquery_embedding = tf.reshape(tf.reduce_mean(lambda_1*tweet_query_word + lambda_1*tweet_query_char,axis=0),shape=[1,embedding_size])

  norm_query = tf.sqrt(tf.reduce_sum(tf.square(tquery_embedding), 1, keep_dims=True))
  tquery_embedding_norm = tquery_embedding / norm_query
  cosine = tf.matmul(tweet_embedding, tquery_embedding_norm, transpose_b=True)
  tweet_query_similarity = tf.reshape(cosine, shape=[tweet_batch_size], name="tweet_query_similarity")

  tquery_embedding_norm_dim = tf.reshape(tquery_embedding_norm, shape=[1,embedding_size])
  query_need_embedding = tf.reshape(tf.reduce_mean(tf.nn.embedding_lookup(normalized_embeddings, need_constant),axis=0),shape=[1,embedding_size])
  cosine_need = tf.matmul(tquery_embedding_norm_dim, query_need_embedding, transpose_b=True)
  tquery_embedding_reqd = tf.reshape(tquery_embedding_norm_dim - (cosine_need*tquery_embedding_norm_dim),shape=[1,embedding_size])
  # we have the need vector without the need vector
  query_avail_embedding = tf.reshape(tf.reduce_mean(tf.nn.embedding_lookup(normalized_embeddings,avail_constant),axis=0),shape=[1,embedding_size])
  query_norm = tf.sqrt(tf.reduce_sum(tf.square(query_avail_embedding),1,keep_dims=True))
  query_avail_embedding_norm = query_embedding / query_norm
  cosine_avail = tf.matmul(tweet_embedding, query_avail_embedding_norm, transpose_b=True)
  reduced_tweet_embedding = tweet_embedding - (tweet_embedding*cosine_avail)
  match_similarity = tf.reshape(tf.matmul(reduced_tweet_embedding, tquery_embedding_reqd, transpose_b=True),shape=[tweet_batch_size],name="match_similarity")

  # Add variable initializer.
  init = tf.global_variables_initializer()
# loading tweet list in integer marking form
# load more data
expand_count = 3
with tf.Session(graph=graph) as session:
  # We must initialize all variables before we use them.
  init.run()
  count = 0
  print("Initialized")

  generators = [generate_batch, generate_batch_char]
  similarities = [similarity, similarity_char]
  placeholders = [[train_inputs,train_labels],[train_input_chars,train_char_labels]]
  losses = [loss, loss_char]
  optimizers = [optimizer, optimizer_char]
  interval1 = 400
  interval2 = 4000
  datas = [data,char_data]
  data_index = [data_index, char_data_index, buffer_index]
  reverse_dictionaries = [reverse_dictionary, reverse_char_dictionary]
  if query_type == 0:
    query_name = 'Need'
  else :
    query_name == 'Avail'
  print(query_tokens)
  count_ = train_model(session, dataset,query_similarity, query_tokens, query_ints, query_name, word_batch_list, char_batch_list, tweet_word_holder, tweet_char_holder, generators, similarities, num_steps, placeholders,losses, optimizers, interval1, interval2, valid_size, valid_examples, reverse_dictionaries, batch_size, num_skips, skip_window, filename , datas, data_index, tweet_batch_size,  0, True)
  placeholders += [[train_inputs, word_char_embeddings, train_labels]]
  count_ = train_model(session, dataset,query_similarity, query_tokens ,query_ints, query_name, word_batch_list, char_batch_list, tweet_word_holder, tweet_char_holder, generators, similarities, num_steps_roll, placeholders,losses, optimizers, interval1, interval2, valid_size, valid_examples, reverse_dictionaries, batch_size, num_skips, skip_window, filename, datas, data_index, tweet_batch_size, count_, True)

  char_batch_dict, word_batch_dict,data, count, dictionary, reverse_dictionary, word_max_len, char_max_len, vocabulary_size, char_dictionary, reverse_char_dictionary, data_index, char_data_index, buffer_index, batch_list, char_batch_list, word_batch_list, char_data = build_everything("complete", consider=True, consider_set="italy")
  expanded_query_tokens, expanded_query_holder, final_query_similarity= expand_query(expand_flag, session,query_ints, np.array(query_tokens),dataset ,similarity_query, word_batch_dict, 100, query_ints, expanded_query_ints, query_similarity, expanded_query_similarity, expand_start_count, expand_count)
  expanded_query_tokens = query_tokens + expanded_query_tokens
  print(expanded_query_tokens)

  count_ = train_model(session, dataset,final_query_similarity, expanded_query_tokens, expanded_query_holder, query_name, word_batch_list, char_batch_list, tweet_word_holder, tweet_char_holder, generators, similarities, num_steps_train , placeholders,losses, optimizers, interval1, interval2, valid_size, valid_examples, reverse_dictionaries, batch_size, num_skips, skip_window, filename , datas, data_index, tweet_batch_size, count_)
  folder_name = './%s/%s/'%(dataset, query_type)
  final_embeddings = normalized_embeddings.eval()
  final_char_embedding = normalized_char_embeddings.eval()
  np.save('../results/%s/%s/%s_word_embeddings.npy'%(dataset, query_name, filename), final_embeddings)
  np.save('../results/%s/%s/%s_char_embeddings.npy'%(dataset, query_name, filename), final_char_embedding)
  saver.save('../results/%s/%s/%s_model.ckpt'%(dataset, query_name, filename))


text:
import pytest
import numpy as np
from numpy.testing import assert_allclose
from .test_fit import MyDataset
from ..datasets import Datasets

@pytest.fixture(scope="session")
def datasets():
    return Datasets([MyDataset(), MyDataset()], mask=np.array([True]))

class TestDatasets:
    @staticmethod
    def test_types(datasets):
        assert datasets.is_all_same_type

    @staticmethod
    def test_likelihood(datasets):
        likelihood = datasets.likelihood()
        assert_allclose(likelihood, 0)

    @staticmethod
    def test_str(datasets):
        assert "MyDataset: 2" in str(datasets)

ext: py

max_stars_repo_name: watsonjj/gammapy

max_issues_repo_name: watsonjj/gammapy

max_forks_repo_name: watsonjj/gammapy

reason: import numpy,from numpy


text:
import unittest
import numpy
import fern
import fern.feature as ff

python_type_to_value_type = {
    int: fern.int64,
    float: fern.float64
}

def no_data_value(
        value_type):
    no_data_value_by_value_type = {
        fern.int8: numpy.iinfo(numpy.int8).min,
        fern.uint8: numpy.iinfo(numpy.uint8).max,
        fern.int32: numpy.iinfo(numpy.int32).min,
        fern.uint32: numpy.iinfo(numpy.uint32).max,
        fern.int64: numpy.iinfo(numpy.int64).min,
        fern.uint64: numpy.iinfo(numpy.uint64).max,
        fern.float32: numpy.finfo(numpy.float32).min,
        fern.float64: numpy.finfo(numpy.float64).max,
    }
    return no_data_value_by_value_type[value_type]

class TestCase(unittest.TestCase):

    @staticmethod
    def masked_raster(
            values,
            mask,
            origin=(0.0, 0.0),
            cell_sizes=(1.0, 1.0),
            value_type=None):
        if value_type is None:
            value_type = python_type_to_value_type[type(values[0][0])]

        return ff.MaskedRaster(values, mask, origin, cell_sizes, value_type)

    def assertMaskedRasterEqual(self,
            raster_we_got,
            raster_we_want):
        self.assertEqual(type(raster_we_got), ff.MaskedRaster)
        self.assertEqual(raster_we_got.value_type, raster_we_want.value_type)
        self.assertEqual(raster_we_got.sizes[0], raster_we_want.sizes[0])
        self.assertEqual(raster_we_got.sizes[1], raster_we_want.sizes[1])
        self.assertEqual(raster_we_got.cell_sizes, raster_we_want.cell_sizes)
        self.assertEqual(raster_we_got.origin, raster_we_want.origin)

        values_we_got = ff.raster_as_numpy_array(raster_we_got)
        values_we_want = ff.raster_as_numpy_array(raster_we_want)

        self.assertEqual(values_we_got.shape, values_we_want.shape)
        self.assertEqual(len(values_we_got.shape), 2)

        nr_rows = values_we_got.shape[0]
        nr_cols = values_we_got.shape[1]
        value_type = raster_we_got.value_type

        for row in xrange(nr_rows):
            for col in xrange(nr_cols):

                if value_type in [fern.float32, fern.float64]:
                    self.assertAlmostEqual(values_we_got[row][col],
                        values_we_want[row][col],
                        msg="{} != {} for cell {}, {}".format(
                            values_we_got[row][col],
                            values_we_want[row][col], row, col))
                else:
                    self.assertEqual(values_we_got[row][col],
                        values_we_want[row][col],
                        msg="{} != {} for cell {}, {}".format(
                            values_we_got[row][col],
                            values_we_want[row][col], row, col))

max_stars_repo_name: quanpands/wflow

max_issues_repo_name: quanpands/wflow

max_forks_repo_name: quanpands/wflow

reason: import numpy


text:
import numpy as np

def get_equiv(equivlence):
    for i in reversed(list(equivlence)):
        for j in equivlence[i]:
            if equivlence[j]:
                equivlence[i] = list(set(equivlence[i] + equivlence[j]))
    # print equivlence
    label = [i for i in equivlence]

    seg = {}
    while label:
        a = label.pop(0)
        for i in equivlence[a]:
            seg[i] = a
            if i in label: label.pop(label.index(i))
    return seg

##################### Image segementing by 8 connected ############
def get_8connected(thresh):
    h,w=thresh.shape
    image_label = np.zeros((h,w), dtype=np.int)
    label = 0
    equivlence = {}
    for i in range(h):
        for j in range(w):
            # if i == 83 and j == 1980:
            #     print
            if thresh[i,j] == 255:
                if i != 0 and j > 0 and j < w-1 :
                    if int(thresh[i - 1, j - 1]) + thresh[i - 1, j] + thresh[i - 1, j + 1] + thresh[i, j - 1] == 0:
                        label += 1
                        equivlence[label] = []
                        image_label[i,j] = label
                    else:
                        l = sorted(list(set([image_label[i - 1, j - 1], image_label[i - 1, j], image_label[i - 1, j + 1], image_label[i, j - 1]])))
                        if l[0] == 0: l.pop(0)
                        try:
                            image_label[i, j] = l[0]
                        except IndexError:
                            print "label error !!!!!!!!!!!",i,j
                        if len(l) > 1:
                            equivlence[l[0]] = list(set(equivlence[l[0]] + l[1:]))
                elif j==0:
                    if int(thresh[i - 1, j]) + thresh[i - 1, j + 1] == 0:
                        label += 1
                        equivlence[label] = []
                        image_label[i, j] = label
                    elif image_label[i - 1, j] == 0:
                        image_label[i, j] = image_label[i - 1, j + 1]
                    elif image_label[i - 1, j + 1] == 0:
                        image_label[i, j] = image_label[i - 1, j + 1]
                    else:
                        if thresh[i - 1, j] == thresh[i - 1, j + 1]:
                            image_label[i, j] = image_label[i - 1, j + 1]
                        else:
                            image_label[i, j] = image_label[i - 1, j]
                            equivlence[image_label[i - 1, j]] = list(set(equivlence[image_label[i - 1, j]]+[image_label[i - 1, j+1]]))
                elif i==0:
                    if thresh[i, j - 1] == 255:
                        image_label[i, j] = image_label[i,j-1]
                    else:
                        label += 1
                        equivlence[label] = []
                        image_label[i, j] = label
                elif j == w-1 and i != 0:
                    if int(thresh[i - 1, j - 1]) + thresh[i - 1, j] + thresh[i, j - 1] == 0:
                        label += 1
                        equivlence[label] = []
                        image_label[i,j] = label
                    else:
                        l = sorted(list(set([image_label[i - 1, j - 1], image_label[i - 1, j], image_label[i, j - 1]])))
                        if l[0] == 0: l.pop(0)
                        if not l:
                            continue
                        try:
                            image_label[i, j] = l[0]
                        except IndexError:
                            print "label error !!!!!!!!!!!",i,j
                        if len(l) > 1:
                            equivlence[l[0]] = list(set(equivlence[l[0]] + l[1:]))
    seg = get_equiv(equivlence)
    # print "seg :",seg
    for i in range(h):
        for j in range(w):
            if image_label[i,j]:
                if image_label[i,j] in seg:
                    image_label[i,j] = seg[image_label[i,j]]
    return image_label
########################################################################################

if __name__ == "__main__":
    thresh = np.array(
        [
            [0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
            [255, 255, 255, 255, 0, 0, 255, 0, 0, 0, 0, 0, 255, 255, 255, 0, 0, 0, 0, 255, 0, 255],
            [255, 255, 255, 255, 255, 0, 255, 255, 255, 0, 0, 0, 0, 0, 255, 255, 255, 255, 0, 255, 0, 0],
            [255, 255, 255, 255, 255, 255, 255, 255, 255, 0, 0, 0, 0, 0, 255, 255, 255, 0, 0, 0, 255, 0],
            [0, 0, 255, 255, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255, 0, 0, 255, 255],
            [0, 0, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 0, 0, 0, 255, 0],
            [0, 0, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 0, 0, 0, 255, 255],
            [0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
            [255, 255, 255, 255, 0, 0, 255, 0, 0, 0, 0, 0, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0],
            [255, 255, 255, 255, 255, 0, 255, 255, 255, 255, 0, 0, 0, 0, 255, 255, 255, 255, 0, 0, 0, 0],
            [255, 255, 255, 255, 255, 255, 255, 255, 255, 0, 255, 255, 0, 0, 255, 255, 255, 0, 0, 0, 0, 0],
            [0, 0, 255, 255, 255, 255, 255, 0, 0, 0, 0, 0, 255, 0, 0, 255, 255, 255, 0, 0, 0, 0],
            [0, 0, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255, 255, 0, 0, 0, 255, 255],
            [0, 0, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 0, 0, 0, 255, 255],
            [0, 0, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 0, 255, 255, 255],
        ]
        , dtype=np.uint8
    )
    print get_8connected(thresh)


text:
import gc

import torch as th
import pickle
import random
import numpy as np
import time
from model_abiliation import DTIModelWithoutBatching as TWIRLS
from model_abiliation import DTITAG

from torch.optim.lr_scheduler import ExponentialLR

from model import DTIModelWithoutBatching, DTIModelWithoutProteinSequence, DTIModel, RMDL
from tqdm import tqdm
from dgl import batch, unbatch
from sklearn.metrics import balanced_accuracy_score, roc_auc_score, precision_recall_curve, \
    average_precision_score, f1_score, auc, recall_score, precision_score
gpu = 0
device = th.device(gpu if th.cuda.is_available() else "cpu")
#device = th.device("cpu")

def get_roce(predList, targetList, roceRate):
    p = sum(targetList)
    n = len(targetList) - p
    predList = [[index, x] for index, x in enumerate(predList)]
    predList = sorted(predList, key=lambda x:x[1], reverse=True)
    tp1 = 0
    fp1 = 0
    maxIndexs = []
    for x in predList:
        if(targetList[x[0]] == 1):
            tp1 += 1
        else:
            fp1 += 1
            if(fp1>((roceRate*n)/100)):
                break
    roce = (tp1*n)/(p*fp1)
    return roce

#with open("train_new_dude_balanced_all2_active.pkl", "rb") as fp:
#    active = pickle.load(fp)

#with open("train_new_dude_balanced_all2_decoy.pkl", "rb") as fp:
#    inactive = pickle.load(fp)

print("load_done")
#random_selector = np.random.randint(len(inactive) - len(active))
#a = int(len(inactive) / (len(active)))
#ds = active + inactive
with open("human_part_train.pkl", 'rb') as fp:
    ds = pickle.load(fp)

random.shuffle(ds)

X = [i[0] for i in ds]
y = [i[1][0] for i in ds]
# random.shuffle(active)
# random.shuffle(inactive)
#
# X_inactive = [i[0] for i in inactive]
# y_inactive = [i[1][0] for i in inactive]

#with open("test_new_dude_all_active_none_pdb.pkl", "rb") as fp:
#    active = pickle.load(fp)

#with open("test_new_dude_all_decoy_none_pdb.pkl", "rb") as fp:
#    inactive = pickle.load(fp)
#
#ds_test = active + inactive
#random.shuffle(ds_test)
with open("human_part_test.pkl", 'rb') as fp:
    ds_test = pickle.load(fp)
#
X_test = [i[0] for i in ds_test]
y_test = [i[1][0] for i in ds_test]

with open("human_part_val.pkl", 'rb') as fp:
    ds_val = pickle.load(fp)

X_val = [i[0] for i in ds_val]
y_val = [i[1][0] for i in ds_val]

#ds_all = ds + ds_test + ds_val
#random.shuffle(ds_all)
#ds = ds_all[0:int(len(ds_all)*0.8)]
#ds_val = ds_all[int(len(ds_all)*0.8): int(len(ds_all)*0.9)]
#ds_test = ds_all[int(len(ds_all)*0.9): int(len(ds_all))]
#X = [i[0] for i in ds]
#y = [i[1][0] for i in ds]
#X_val = [i[0] for i in ds_val]
#y_val = [i[1][0] for i in ds_val]
#X_test = [i[0] for i in ds_test]
#y_test = [i[1][0] for i in ds_test]
model = DTITAG()
#model.load_state_dict(th.load('without_batching2021_09_05-20_23_26-26_checkpoint.pt', map_location=th.device('cpu'))['net_state'])
model.to(device)
MODEL_NAME = f"model-{int(time.time())}"
optimizer = th.optim.Adam(model.parameters(), lr=1e-3)
criterion = th.nn.BCELoss()
scheduler = ExponentialLR(optimizer, gamma=0.90)

print("init done")

def fwd_pass(X, y, train=False):
    if train:
        model.zero_grad()
    out = []

    for item in X:
        x = [0, 0]
        x[0] = item[0].to(device)
        x[1] = item[1].to(device)
        out.append(model(x))
        del x

    out = th.stack(out, 0).view(-1, 1).to(device)

    y = th.Tensor(y).view(-1, 1).to(device)

    loss = criterion(out, y)

    matches = [th.round(i) == th.round(j) for i, j in zip(out, y)]
    acc = matches.count(True) / len(matches)

    if train:
        loss.backward()
        optimizer.step()

    return acc, loss, out

def test_func(model_f, y_label, X_test_f):
    y_pred = []
    y_label = th.Tensor(y_label)
    print("Testing:")
    print("-------------------")
    with tqdm(range(0, len(X_test_f), 1)) as tepoch:
        for i in tepoch:
            with th.no_grad():
                x = [0, 0]
                x[0] = X_test_f[i][0].to(device)
                x[1] = X_test_f[i][1].to(device)
                y_pred.append(model_f(x).cpu())

    y_pred = th.cat(y_pred, dim=0)
    y_pred_c = [round(i.item()) for i in y_pred]
    roce1 = get_roce(y_pred, y_label, 0.5)
    roce2 = get_roce(y_pred, y_label, 1)
    roce3 = get_roce(y_pred, y_label, 2)
    roce4 = get_roce(y_pred, y_label, 5)
    print("AUROC: " + str(roc_auc_score(y_label, y_pred)), end=" ")
    print("PRAUC: " + str(average_precision_score(y_label, y_pred)), end=" ")
    print("F1 Score: " + str(f1_score(y_label, y_pred_c)), end=" ")
    print("Precision Score:" + str(precision_score(y_label, y_pred_c)), end=" ")
    print("Recall Score:" + str(recall_score(y_label, y_pred_c)), end=" ")
    print("Balanced Accuracy Score " + str(balanced_accuracy_score(y_label, y_pred_c)), end=" ")
    print("0.5 re Score " + str(roce1), end=" ")
    print("1 re Score " + str(roce2), end=" ")
    print("2 re Score " + str(roce3), end=" ")
    print("5 re Score " + str(roce4), end=" ")
    print("-------------------")

def train(net):
    EPOCHS = 100
    BATCH_SIZE = 80

    with open("model.log", "a") as f:
        for epoch in range(EPOCHS):
            losses = []
            accs = []
            with tqdm(range(0, len(X), BATCH_SIZE)) as tepoch:
                for i in tepoch:
                    tepoch.set_description(f"Epoch {epoch + 1}")
                    try:

                        batch_X = X[i: i+BATCH_SIZE]
                        batch_y = y[i: i+BATCH_SIZE]
                    except:
                        gc.collect()
                        continue
                    acc, loss, _ = fwd_pass(batch_X, batch_y, train=True)

                    losses.append(loss.item())
                    accs.append(acc)
                    acc_mean = np.array(accs).mean()
                    loss_mean = np.array(losses).mean()
                    tepoch.set_postfix(loss=loss_mean, accuracy=100. * acc_mean)
                    if i % 100000 == 0:
                        test_func(model, y_test, X_test)
                        test_func(model, y_val, X_val)
                        # print(f'Average Loss: {val_loss}')
                        # print(f'Average Accuracy: {val_acc}')
                        f.write(
                            f"{MODEL_NAME},{round(time.time(), 3)},{round(float(acc), 2)},{round(float(loss), 4)}\n")
                scheduler.step()
            print(f'Average Loss: {np.array(losses).mean()}')
            print(f'Average Accuracy: {np.array(accs).mean()}')
            #dt = time.strftime("%Y_%m_%d-%H_%M_%S")
            #fn = "without_batching" + str(dt) + str("-") + \
            #     str(epoch) + "_checkpoint.pt"
            #info_dict = {
            #    'epoch': epoch,
            #    'net_state': model.state_dict(),
            #    'optimizer_state': optimizer.state_dict()
            #}
            #th.save(info_dict, fn)

train(model)
test_func(model, y_test, X_test)

max_stars_repo_name: yazdanimehdi/AttentionSiteDTI

max_issues_repo_name: yazdanimehdi/AttentionSiteDTI

max_forks_repo_name: yazdanimehdi/AttentionSiteDTI


text:
#!/usr/bin/env python3
#
import networkx

def manhattan(p1, p2):
    return sum(abs(a-b) for a, b in zip(p1, p2))

def main():
    points = set()
    # with open('sinput', 'r') as fp:
    with open('input', 'r') as fp:
        for line in fp.read().split('\n'):
            points.add(tuple(map(int, line.split(","))))

    N = networkx.Graph()
    for p1 in points:
        for p2 in points:
            if manhattan(p1, p2) <= 3:
                N.add_edge(p1, p2)
    print(networkx.number_connected_components(N))

if __name__ == '__main__':
    main()

max_stars_repo_name: smolsbs/aoc

max_issues_repo_name: smolsbs/aoc

max_forks_repo_name: smolsbs/aoc

reason: import networkx


text:
import random

from tqdm import tqdm

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

from vp_suite.base.base_model import VideoPredictionModel
from vp_suite.model_blocks import Autoencoder
from vp_suite.model_blocks.predrnn import SpatioTemporalLSTMCell, ActionConditionalSpatioTemporalLSTMCell
from vp_suite.model_blocks.phydnet import PhyCell_Cell, K2M

class STPhy(VideoPredictionModel):
    r"""

    """

    # model-specific constants
    NAME = "ST-Phy"
    CAN_HANDLE_ACTIONS = True

    # model hyperparameters
    num_layers = 3  #: TODO
    phycell_kernel_size = (7, 7)  #: TODO
    phycell_channels = 49  #: TODO
    st_cell_channels = 64  #: TODO
    inflated_action_dim = 3  #: TODO

    # model training hyperparams
    reconstruction_loss_scale = 0.1  #: TODO
    decoupling_loss_scale = 100.0  #: TODO
    moment_loss_scale = 1.0  #: TODO
    teacher_forcing_decay = 0.003

    def __init__(self, device, **model_kwargs):
        r"""

        Args:
            device ():
            **model_kwargs ():
        """
        super(STPhy, self).__init__(device, **model_kwargs)

        self.dim_st_hidden = [self.st_cell_channels] * self.num_layers
        self.dim_phy_hidden = [self.phycell_channels] * self.num_layers

        self.autoencoder = Autoencoder(self.img_shape, self.st_cell_channels, self.device)
        _, _, self.enc_h, self.enc_w = self.autoencoder.encoded_shape
        self.recurrent_cell = SpatioTemporalLSTMCell

        if self.action_conditional:
            self.recurrent_cell = ActionConditionalSpatioTemporalLSTMCell
            self.action_inflate = nn.Linear(in_features=self.action_size,
                                            out_features=self.inflated_action_dim * self.enc_h * self.enc_w,
                                            bias=False)
            self.action_conv_h = nn.Conv2d(in_channels=self.inflated_action_dim, out_channels=self.st_cell_channels,
                                           kernel_size=(5, 1), padding=(2, 0), bias=False)
            self.action_conv_w = nn.Conv2d(in_channels=self.inflated_action_dim, out_channels=self.st_cell_channels,
                                           kernel_size=(1, 5), padding=(0, 2), bias=False)

        st_cells, phycells, hidden_convs = [], [], []
        for i in range(self.num_layers):
            cell_in_channel = self.dim_st_hidden[0] if i == 0 else self.dim_st_hidden[i - 1]
            st_cells.append(self.recurrent_cell(cell_in_channel, self.dim_st_hidden[i], self.enc_h, self.enc_w,
                                                filter_size=5, stride=1, layer_norm=True))

            phycells.append(PhyCell_Cell(input_dim=cell_in_channel, action_conditional=self.action_conditional,
                                         action_size=self.action_size, hidden_dim=self.dim_phy_hidden[i],
                                         kernel_size=self.phycell_kernel_size).to(self.device))

            hc_bias = i < self.num_layers - 1
            hidden_convs.append(nn.Conv2d(in_channels=self.st_cell_channels + self.dim_st_hidden[i],
                                          out_channels=self.st_cell_channels, kernel_size=(1, 1), bias=hc_bias))

        self.st_cell_list = nn.ModuleList(st_cells)
        self.phycell_list = nn.ModuleList(phycells)
        self.hidden_conv_list = nn.ModuleList(hidden_convs)

        # shared adapter
        adapter_dim_hidden = self.dim_st_hidden[0]
        self.adapter = nn.Conv2d(adapter_dim_hidden, adapter_dim_hidden, 1, stride=1, padding=0, bias=False)

        self.constraints = torch.zeros((self.phycell_channels, *self.phycell_kernel_size), device=self.device)
        ind = 0
        for i in range(0, self.phycell_kernel_size[0]):
            for j in range(0, self.phycell_kernel_size[1]):
                self.constraints[ind, i, j] = 1
                ind += 1

    def pred_1(self, x, **kwargs):
        r"""

        Args:
            x ():
            **kwargs ():

        Returns:

        """
        return self(x, pred_frames=1, **kwargs)[0].squeeze(dim=1)

    def forward(self, x, pred_frames=1, **kwargs):
        r"""

        Args:
            x ():
            pred_frames ():
            **kwargs ():

        Returns:

        """
        # in training mode (default: False), returned sequence starts with 2nd context frame,
        # and the moment regularization loss is calculated.
        train = kwargs.get("train", False)
        teacher_forcing = kwargs.get("teacher_forcing", False) and train
        batch_size, context_frames = x.shape[:2]
        if train:  # during training, the whole sequence is passed
            context_frames -= pred_frames
        empty_actions = torch.zeros(x.shape[0], context_frames+pred_frames-1, device=self.device)
        actions = kwargs.get("actions", empty_actions)
        if self.action_conditional:
            if actions.equal(empty_actions) or actions.shape[-1] != self.action_size:
                raise ValueError("Given actions are None or of the wrong size!")

        # init ST and Phy
        phy_h_t = []
        st_h_t = []
        st_c_t = []
        delta_c_list = []
        delta_m_list = []

        decouple_loss = []

        for i in range(self.num_layers):
            zeros = torch.zeros([batch_size, self.dim_st_hidden[i], self.enc_h, self.enc_w]).to(self.device)
            st_h_t.append(zeros)
            st_c_t.append(zeros)
            delta_c_list.append(zeros)
            delta_m_list.append(zeros)
            phy_h_t.append(torch.zeros(batch_size, self.st_cell_channels, self.enc_h, self.enc_w).to(self.device))

        st_memory = torch.zeros([batch_size, self.dim_st_hidden[0], self.enc_h, self.enc_w]).to(self.device)
        out_frames = []
        x_gen = None

        # fwd prop through time
        for t in range(context_frames + pred_frames - 1):

            # get input, depending on training/inference stage
            if t < context_frames or teacher_forcing:
                next_input = self.autoencoder.encode(x[:, t])
            else:
                next_input = x_gen

            for i, (st_cell, phycell) in enumerate(zip(self.st_cell_list, self.phycell_list)):

                # phy
                phy_h_t[i] = phycell(next_input, actions[:, t], phy_h_t[i])  # returns (hidden, hidden)

                # st
                if self.action_conditional:
                    ac = self.action_inflate(actions[:, t]).view(-1, self.inflated_action_dim, self.enc_h, self.enc_w)
                    inflated_action = self.action_conv_h(ac) + self.action_conv_w(ac)
                    st_h_t[i], st_c_t[i], st_memory, delta_c, delta_m \
                        = st_cell(next_input, st_h_t[i], st_c_t[i], st_memory, inflated_action)
                else:
                    st_h_t[i], st_c_t[i], st_memory, delta_c, delta_m \
                        = st_cell(next_input, st_h_t[i], st_c_t[i], st_memory)
                delta_c_list[i] = F.normalize(self.adapter(delta_c).view(delta_c.shape[0], delta_c.shape[1], -1), dim=2)
                delta_m_list[i] = F.normalize(self.adapter(delta_m).view(delta_m.shape[0], delta_m.shape[1], -1), dim=2)

                # merge
                x_gen = self.hidden_conv_list[i](torch.cat([st_h_t[i], phy_h_t[i]], dim=-3))

            if train or t >= (context_frames - 1):
                out_frame = self.autoencoder.decode(x_gen)
                out_frames.append(out_frame)

            # Spatio-temporal decoupling loss in STCell during training
            if train:
                for i in range(0, self.num_layers):
                    decouple_loss.append(
                        torch.mean(torch.abs(torch.cosine_similarity(delta_c_list[i], delta_m_list[i], dim=2))))

        out_frames = torch.stack(out_frames, dim=1)

        if train:
            # Moment regularization loss during training
            k2m = K2M(self.phycell_kernel_size).to(self.device)
            moment_loss = 0
            for b in range(0, self.phycell_list[0].input_dim):
                filters = self.phycell_list[0].F.conv1.weight[:, b]
                moment = k2m(filters.double()).float()
                moment_loss += torch.mean(self.moment_loss_scale * (moment - self.constraints) ** 2)
            model_losses = {
                "moment regularization loss": moment_loss,
                "memory decoupling loss": torch.mean(torch.stack(decouple_loss, dim=0)),
            }
        else:
            model_losses = None
        return out_frames, model_losses

    def train_iter(self, config, data_loader, optimizer, loss_provider, epoch):
        r"""

        Args:
            config ():
            data_loader ():
            optimizer ():
            loss_provider ():
            epoch ():

        Returns:

        """
        teacher_forcing_ratio = np.maximum(0, 1 - epoch * self.teacher_forcing_decay)
        loop = tqdm(data_loader)
        for batch_idx, data in enumerate(loop):

            # fwd
            input_frames, _, actions = self.unpack_data(data, config, complete=True)
            teacher_forcing = True if random.random() < teacher_forcing_ratio else False
            predictions, model_losses = self(input_frames, pred_frames=config["pred_frames"],
                                             actions=actions, train=True, teacher_forcing=teacher_forcing)

            # loss
            targets = input_frames[:, 1:]
            _, total_loss = loss_provider.get_losses(predictions, targets)
            if model_losses is not None:
                for value in model_losses.values():
                    total_loss += value

            optimizer.zero_grad()
            total_loss.backward()
            optimizer.step()

            loop.set_postfix(loss=total_loss.item())


text:
#   supplementing modifiedMexicanHatTest5.py
#   outputing the charts, given the results
import numpy as np
import matplotlib.pyplot as plt
from armor import pattern
from armor import defaultParameters as dp
dbz = pattern.DBZ
DS  = pattern.DBZstream

dataFolder  = dp.root + "labLogs/2014-5-2-modifiedMexicanHatTest5/"
outputFolder= dataFolder
WRFnames    = [ "WRF"+("0"+str(v))[-2:] for v in range(1,21)]

sigmas      = [1, 2, 4, 5, 8 ,10 ,16, 20, 32, 40, 64, 80, 128, 160, 256, 320,]
allWRFsStreamMean  = 0.
dbzCount    = 0

for WRFname in WRFnames:
    ds  = DS(dataFolder=dataFolder,
                 name="rainband_march2014" + WRFname,
                 outputFolder="",
                 imageFolder="",
                 key1=WRFname,               # keywords to pick out specific files
                 key2="LOGspec.dat",
                 key3="WRF_Rainband", #safety check
                 preload=True,
                 imageExtension = '.png',     #added 2013-09-27
                 dataExtension  = '.dat',
                 )
    print "\n==================\nSaving histograms for ", ds.name
    for dbzpattern in ds:
        dbzCount    += 1
        streamMeanUpdate = np.array([(dbzpattern.matrix==v).sum()   for v in sigmas])
        allWRFsStreamMean  = 1.* ((allWRFsStreamMean*(dbzCount -1)) + streamMeanUpdate ) / dbzCount
        histogramName   = "kongreywrf" + dbzpattern.dataTime + WRFname + "_LOGspec_histogram"+ ds.imageExtension
        print dbzpattern.name, "->", histogramName
        plt.clf()
        dbzpattern.histogram(display=False, outputPath=outputFolder+histogramName)

plt.close()
plt.plot(sigmas, allWRFsStreamMean)
plt.title(ds.name + '- average laplacian-of-gaussian max-response spectrum for ' +str(dbzCount) + 'WRF patterns')
plt.savefig(outputFolder + ds.name + "_all_wrfs_average_LoG_max_response spectrum.png")
plt.close()

"""
#   run modifiedMexicanHatTest6a.py and then:
allWRFsStreamMean   = array([ 2562.4375,   655.5625,   526.15  ,   741.51  ,   858.6425,
                            1457.79  ,  1710.095 ,  2971.355 ,  3561.9125,  4406.915 ,
                            1488.0375,    59.5925,     0.    ,     0.    ,     0.    ,     0.    ])

streamMeanCOMPREF   = streamMean
sigmas  = np.array(sigmas)
plt.close()
plt.plot(sigmas, streamMeanCOMPREF)
plt.plot(sigmas[:-4]*4, allWRFsStreamMean[:-4]*16)
plt.title("COMPREF and WRFs mean max-response LOG spectra from Kong-Rey data")
plt.show()
"""

max_stars_repo_name: yaukwankiu/armor

max_issues_repo_name: yaukwankiu/armor

max_forks_repo_name: yaukwankiu/armor


text:
import cv2
import numpy as np
import visvis as vv

import data_loader
from utils import get_args

def visualize(im, vol, sz=0.25, thr=0.99):
    im, vol = im.numpy(), vol.numpy()
    print("Image shape:", im.shape)
    print("Volume shape:", vol.shape)

    # overlap with 3d representation + BGR->RGB
    im = im.transpose(1, 2, 0)  # H,W,C
    im = im[:, :, ::-1]
    im = cv2.resize(im, (128, 128))

    t = vv.imshow(im)
    t.interpolate = True  # interpolate pixels

    # volshow will use volshow3 and rendering the isosurface if OpenGL version is >= 2.0
    # Otherwise, it will show slices with bars that you can move (much less useful).
    im = (im * 128 + 128).astype(np.uint8)
    # im = np.ones_like(im)

    volRGB = np.stack(((vol >= thr) * im[:, :, 0],
                       (vol >= thr) * im[:, :, 1],
                       (vol >= thr) * im[:, :, 2]), axis=3)

    v = vv.volshow(volRGB, renderStyle='iso')
    v.transformations[1].sz = sz  # Z rescaling

    l0 = vv.gca()
    l0.light0.ambient = 0.9  # 0.2 is default for light 0
    l0.light0.diffuse = 1.0  # 1.0 is default

    a = vv.gca()
    a.axis.visible = 0
    a.camera.fov = 0  # orthographic
    vv.use().Run()

if __name__ == "__main__":
    args = get_args()

    trainset = data_loader.FacesWith3DCoords(
        images_dir=args.images_dir, mats_dir=args.mats_dir, landmarks_dir=args.lands_dir, transform=args.transform
    )

    i = np.random.randint(len(trainset))
    img, volume, _ = trainset[i]
    visualize(img, volume)


text:
# File I/O module designed to make manipulation easier for:
# - DX volumetric files
import numpy as np

# https://stackoverflow.com/questions/434287/what-is-the-most-pythonic-way-to-iterate-over-a-list-in-chunks
def chunker(seq, size):
    return (seq[pos:pos + size] for pos in range(0, len(seq), size))

# In this form, data is treated as a 3D numpy matrix.
# It needs to convert between linear representation of DX files into the 3D internally.
# The default volume units of VMD is in Angstroms, so if we wish to transform the data
# then we specify a conversion scale.
# Note, this returns data in Angstroms as well.
# Uses by default Crow-major order. This is C-like..?
def read_from_dx(fname, units='A'):
    selfname='dxio.py - read_from_dx'
    if units=='A':
        scale=0.1
    elif units=='nm':
        scale=1.0
    else:
        print( '= = ERROR in %s: Units argument of write_to_dx accepts only \'nm\' and \'A\'!' % selfname )
        return

    #Initialise variables.
    dims = np.zeros(3)
    orig = np.zeros(3)
    abc  = np.zeros((3,3))
    bHeader = True
    deltadim = 0
    count = 0
    with open(fname) as fp:
        for line in fp:
            #Ignore comments and empty lines
            if len(line)==0 or line[0] == '#':
                continue
            l = line.split()
            if bHeader:
                if l[0] == 'origin':
                    orig = np.multiply(scale, np.array(map(float, l[1:4])))
                    print( '= = Input origin: ', orig )
                elif l[0] == 'object':
                    if l[1] == '1':
                        dims = np.array(map(int, l[-3:]))
                        print( '= = Input dimension: ', dims )
                    if l[1] == '3':
                        ntot = int(l[-3])
                        print( '= = Expected total data points: ', ntot )
                        if ntot != dims[0]*dims[1]*dims[2]:
                            print( '= = ERROR in %s: Total data points is not equal to dimensions!?' % selfname )
                            return
                        else:
                            data = np.zeros(ntot)
                            bHeader = False
                elif l[0] == 'delta':
                    abc[deltadim] = np.multiply(scale, np.array(map(float, l[1:4])))
                    deltadim=deltadim+1
            else:
                #Data entries
                if count < ntot:
                    for a in range(len(l)):
                        data[count]=float(l[a])
                        count=count+1
                else:
                    if l == []:
                        continue
                    if l[0] == 'object' and l[-1] == 'field':
                        print( l[:] )

    data_order='C'
    data = np.multiply(1.0/scale**3, np.reshape(data, dims, order=data_order))

    return [data, dims, orig, abc]

#Here the data format read:
#    - dims: number of points in each dimension.
#    - orig: the minimum in each dimensions.
#    - abc: unit cell vectors, or bindwith in each dimension. Needs to be (3,3)
def write_to_dx(fname, data, dims, orig, abc, units='A', bScaleDat=True):
    selfname='dxio.py - write_to_dx'
    fp = open(fname, 'w')
    if units=='A':
        scale=10.0
    elif units=='nm':
        scale=1.0
    else:
        print( '= = ERROR in %s: units argument of write_to_dx accepts only \'nm\' and \'A\'!' % selfname )
        return
    #Sanity checks.
    if dims[0] != data.shape[0] or dims[1] != data.shape[1] or dims[2] != data.shape[2]:
        print( '= = ERROR in %s: Data dimensions do not match with the matrix dimenstions!' % selfname )
        print( dims, data.shape )
        outorig=np.multiply(scale,orig)
        return

    # Check if abc is given as a matrix oa sert of vectors.
    outabc =np.multiply(scale,abc)
    if len(outabc.shape)==1:
        outabc=np.array([abc[0],0,0],[0,abc[1],0],[0,0,abc[2]])
    outorig=np.multiply(scale,orig)
    #Writing headers
    print( '#DX-file written by dxio.py', file=fp )
    print( 'object 1 class gridpositions counts %i %i %i' % (dims[0], dims[1], dims[2]), file=fp )
    print( 'origin %g %g %g' % (outorig[0], outorig[1], outorig[2]), file=fp )
    for i in range(0,3):
        print( 'delta %g %g %g' % (outabc[i,0], outabc[i,1], outabc[i,2]), file=fp )
    print( 'object 2 class gridpositions counts %i %i %i' % (dims[0], dims[1], dims[2]), file=fp )
    ntot=dims[0]*dims[1]*dims[2]
    print( 'object 3 class array type double rank 0 items %i data follows' % ntot, file=fp )

    #Data files itself. The DX format is actually C-order, so will reformulate F-order matrices.
    if bScaleDat:
        flat = np.multiply(1.0/scale**3,data.flatten(order='C'))
    else:
        flat = data.flatten(order='C')
    #ntot=len(data)
    #print( 'object 3 class array type double rank 0 items %i data follows' % ntot, file=fp )
    for c in chunker(flat, 3):
        print(" ".join("%g" % i for i in c ), file=fp)
    print( '', file=fp )
    print( 'object "density [%s^-3]" class field' % units, file=fp )
    fp.close()

def debug_all_vars(data, dims, orig, abc):
    print( "= = Data dimensions:", data.shape )
    print( "= = Interpreted dimensions:", dims )
    print( "= = Origin:", orig )
    print( "= = Box vectors (below):" )
    print( abc )

    return

max_stars_repo_name: zharmad/SpinRelax

max_issues_repo_name: zharmad/SpinRelax

max_forks_repo_name: zharmad/SpinRelax


text:
import argparse
import os
from util import util
import torch
import json

class BaseOptions:
    def __init__(self):
        self.parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
        self.initialized = False
        self.opt = None

    def initialize(self):
        # config path - overrules other args
        self.parser.add_argument('--config_path', default=None,
                                 help='path to json config file: if given, reads args from given config file '
                                      'and ignores other command line arguments')
        # data params
        self.parser.add_argument('--dataroot', default=None,
                                 help='path to meshes (should have subfolders train, test). '
                                      'must be supplied, either in a config file or as an argument.')
        self.parser.add_argument('--dataset_mode', choices={"classification", "segmentation"}, default='classification')
        self.parser.add_argument('--ninput_edges', type=int, default=750,
                                 help='# of input edges (will include dummy edges)')
        self.parser.add_argument('--max_dataset_size', type=int, default=float("inf"),
                                 help='Maximum number of samples per epoch')
        # network params
        self.parser.add_argument('--batch_size', type=int, default=16, help='input batch size')
        self.parser.add_argument('--arch', type=str, default='mconvnet',
                                 help='selects network to use')  # todo add choices
        self.parser.add_argument('--resblocks', type=int, default=0, help='# of res blocks')
        self.parser.add_argument('--fc_n', type=int, default=100, help='# between fc and nclasses')  # todo make generic
        self.parser.add_argument('--ncf', nargs='+', default=[16, 32, 32], type=int, help='conv filters')
        self.parser.add_argument('--pool_res', nargs='+', default=[1140, 780, 580], type=int, help='pooling res')
        self.parser.add_argument('--norm', type=str, default='batch',
                                 help='instance normalization or batch normalization or group normalization')
        self.parser.add_argument('--num_groups', type=int, default=16, help='# of groups for groupnorm')
        self.parser.add_argument('--init_type', type=str, default='normal',
                                 help='network initialization [normal|xavier|kaiming|orthogonal]')
        self.parser.add_argument('--init_gain', type=float, default=0.02,
                                 help='scaling factor for normal, xavier and orthogonal.')
        # attention params
        self.parser.add_argument('--attn_n_heads', type=int, default=4,
                                 help='number of heads for Multi Headed Attention')
        self.parser.add_argument('--prioritize_with_attention', action='store_true',
                                 help='if given, the priority queue for the pool operation is calculated from the attention softmax.'
                                      'default priority is l2 norm')
        self.parser.add_argument('--attn_dropout', type=float, default=0.1, help='dropout fraction for attention layer')
        self.parser.add_argument('--attn_max_dist', type=int, default=None,
                                 help='max distance for local attention. default (None) is global attention')
        self.parser.add_argument('--attn_use_values_as_is', action='store_true',
                                 help='if given, attention layers learn a weighting of the input features. '
                                      'default behavior is learning a weighting of a linear transformation '
                                      'of the input features.')
        self.parser.add_argument('--double_attention', action='store_true',
                                 help='if given, the edge priorities are calculated using the results of applying the attention layer to the '
                                      'results of itself. default behavior is calculating the priorities from the results of applying the '
                                      'attention to the convolutional features. '
                                      'NOTE: attn_use_values_as_is must be True if you use this option, since the attention layer works on its own outputs.')
        self.parser.add_argument('--attn_use_positional_encoding', action='store_true',
                                 help='use relative positional encodings to add positional meaning to attention. '
                                      'relative position is determined by the number "hops" it takes to reach one edge from another, '
                                      'where hops are only allowed through convolutional neighbors (edges that share the same triangle). '
                                      'mathematically, this is shortest path in a graph where every edge is a node and adjacency '
                                      'is determined in the same way as convolutional neighborhood.')
        self.parser.add_argument('--attn_max_relative_position', type=int, default=6,
                                 help='the maximal relative position for positional encoding. edges further aways than max_pos '
                                      'are treated as if their position is max_pos. a 5-distance-neighborhood of an edge contains '
                                      'about 60 edges (see doc/neighbors_vs_local.png or .csv).')
        # general params
        self.parser.add_argument('--num_threads', default=3, type=int, help='# threads for loading data')
        self.parser.add_argument('--gpu_ids', type=str, default='0', help='gpu ids: e.g. 0  0,1,2, 0,2. use -1 for CPU')
        self.parser.add_argument('--name', type=str, default='debug',
                                 help='name of the experiment. It decides where to store samples and models')
        self.parser.add_argument('--checkpoints_dir', type=str, default='./checkpoints', help='models are saved here')
        self.parser.add_argument('--serial_batches', action='store_true',
                                 help='if true, takes meshes in order, otherwise takes them randomly')
        self.parser.add_argument('--seed', type=int, help='if specified, uses seed')
        # visualization params
        self.parser.add_argument('--export_folder', type=str, default='',
                                 help='exports intermediate collapses to this folder')
        #
        self.initialized = True

    def is_given_config_path(self):
        return self.opt.config_path is not None

    def get_default_options(self):
        return self.parser.parse_args([])

    @staticmethod
    def inhabit_options_from_config_args(opt, config_args):
        for arg_name, arg_value in config_args.items():
            if hasattr(opt, arg_name):
                setattr(opt, arg_name, arg_value)

    def parse_config_file(self):
        opt = self.get_default_options()

        with open(self.opt.config_path, 'r') as f:
            config_args = json.load(f)

        BaseOptions.inhabit_options_from_config_args(opt, config_args)
        return opt

    def raise_if_no_dataroot(self):
        if self.opt.dataroot is None:
            raise argparse.ArgumentError("dataroot must be supplied, either in a config file or as an argument.")

    def parse(self):
        if not self.initialized:
            self.initialize()

        self.opt, unknown = self.parser.parse_known_args()
        if self.is_given_config_path():
            self.opt = self.parse_config_file()

        self.raise_if_no_dataroot()

        self.opt.is_train = self.is_train  # train or test

        str_ids = self.opt.gpu_ids.split(',')
        self.opt.gpu_ids = []
        for str_id in str_ids:
            id = int(str_id)
            if id >= 0:
                self.opt.gpu_ids.append(id)
        # set gpu ids
        if len(self.opt.gpu_ids) > 0:
            torch.cuda.set_device(self.opt.gpu_ids[0])

        args = vars(self.opt)

        if self.opt.seed is not None:
            import numpy as np
            import random
            torch.manual_seed(self.opt.seed)
            np.random.seed(self.opt.seed)
            random.seed(self.opt.seed)

        if self.opt.export_folder:
            self.opt.export_folder = os.path.join(self.opt.checkpoints_dir, self.opt.name, self.opt.export_folder)
            util.mkdir(self.opt.export_folder)

        if self.is_train:
            print('------------ Options -------------')
            for k, v in sorted(args.items()):
                print('%s: %s' % (str(k), str(v)))
            print('-------------- End ----------------')

            # save to the disk
            expr_dir = os.path.join(self.opt.checkpoints_dir, self.opt.name)
            util.mkdir(expr_dir)

            file_name = os.path.join(expr_dir, 'opt.txt')
            with open(file_name, 'wt') as opt_file:
                opt_file.write('------------ Options -------------\n')
                for k, v in sorted(args.items()):
                    opt_file.write('%s: %s\n' % (str(k), str(v)))
                opt_file.write('-------------- End ----------------\n')
        return self.opt


text:
"""
This script provides code for training ICNet from scratch.

Paper can be found here: https://arxiv.org/pdf/1704.08545.pdf
"""

import os
import time
import logging
import datetime

import cv2
import numpy as np
import tensorflow as tf
from tqdm import tqdm

from model import ICNet
from utils import (build_tfrecords_batch, create_loss, prepare_label,
                   read_and_decode_semseg_tfrecord, set_logging_verbosity)

MAX_ITERATIONS = int(1e6)
DEFAULT_NETWORK_LOCATION = "./logs_dir"

# Tensorflow cmd args
FLAGS = tf.flags.FLAGS
tf.flags.DEFINE_integer("batch_size", "16", "batch size for training")
tf.flags.DEFINE_string("logs_dir", DEFAULT_NETWORK_LOCATION, "path to logs directory")
tf.flags.DEFINE_float("learning_rate", "1e-4", "Learning rate for Adam Optimizer")
tf.flags.DEFINE_string('mode', "train", "Mode train/ test/ visualize")
tf.flags.DEFINE_integer('num_samples_per_val', '2000', "Number of samples to consider per validation period")
tf.flags.DEFINE_integer("min_after_dequeue", "100", "How big a buffer we want to sample from data")
tf.flags.DEFINE_string("path_to_records", "./tfrecords", "The path to tfrecords")
tf.flags.DEFINE_string("records_prefix", "ade20k", "The path to tfrecords")
tf.flags.DEFINE_string("dataset", "ade20k", "The path to tfrecords")

if FLAGS.dataset == "ade20k":
    # Void label is set to 0 and we have 150 classes, so we add 1 to
    # number of classes to support void label since we can't easily
    # skip it without preprocessing/remapping classes
    SEMSEG_NUM_CLASSES = 150 + 1
    SEMSEG_SHAPE = (480, 480)
    IGNORE_LABEL = 0
elif FLAGS.dataset == "cityscapes":
    SEMSEG_NUM_CLASSES = 19
    SEMSEG_SHAPE = (1024,2048)
    IGNORE_LABEL = 255
else:
    raise ValueError(
        "Unknown dataset; if training on a custom dataset, you must specify values "
        "for `SEMSEG_NUM_CLASSES`, `SEMSEG_SHAPE`, `IGNORE_LABEL`")

def main():

    set_logging_verbosity(3)

    net = ICNet(SEMSEG_SHAPE, FLAGS.logs_dir, SEMSEG_NUM_CLASSES, FLAGS.learning_rate)

    with net.graph.as_default():

        global_step = tf.Variable(0, name='global_step', trainable=False)
        best_loss = tf.Variable(1e7, name='best_loss', trainable=False, dtype=tf.float32)
        best_iou = tf.Variable(0, name='best_iou', trainable=False, dtype=tf.float32)
        best_px_acc = tf.Variable(0, name='best_px_acc', trainable=False, dtype=tf.float32)

        # Get shortcuts to all placeholders as well as the computation graph
        image = net.placeholders["image"]
        keep_prob = net.placeholders["keep_probability"]
        annotation = net.placeholders["annotation"]
        is_training = net.placeholders["is_training"]

        # Add summary ops for tensorboard
        rgb_image = tf.reverse(image, axis=[3])
        tf.summary.image("input_image", rgb_image, max_outputs=4)
        tf.summary.image(
            "ground_truth", tf.cast(tf.multiply(annotation, 255), tf.uint8), max_outputs=4)

        # Create cascade label guidence based loss expression
        lambda1, lambda2, lambda3 = 0.16, 0.4, 1.0
        sub_4_loss = create_loss(net.low_res_logits, annotation, SEMSEG_NUM_CLASSES, IGNORE_LABEL)
        sub_24_loss = create_loss(net.med_res_logits, annotation, SEMSEG_NUM_CLASSES, IGNORE_LABEL)
        sub_124_loss = create_loss(net.high_res_logits, annotation, SEMSEG_NUM_CLASSES, IGNORE_LABEL)
        loss = lambda1 * sub_4_loss + lambda2 * sub_24_loss + lambda3 * sub_124_loss
        tf.summary.scalar("entropy", loss)

        # Create expression for running mean IoU
        pred_flatten = tf.reshape(net.predict, [-1,])
        label_flatten = tf.reshape(annotation, [-1,])

        mask = tf.not_equal(label_flatten, IGNORE_LABEL)
        indices = tf.squeeze(tf.where(mask), 1)
        gt = tf.cast(tf.gather(label_flatten, indices), tf.int32)
        pred = tf.gather(pred_flatten, indices)

        mIoU, miou_update_op = tf.metrics.mean_iou(
            predictions=pred, labels=gt, num_classes=SEMSEG_NUM_CLASSES, name="mIoU")
        px_acc, px_acc_update_op = tf.metrics.accuracy(predictions=pred, labels=gt, name="px_acc")

        # Group the update ops for both metrics for convenience
        metrics_update_op = tf.group(miou_update_op, px_acc_update_op)

        # Make an operation to reset running variables for mIoU and pixel accuracy so
        # we can reset mIoU for each validation
        running_miou_vars = tf.get_collection(tf.GraphKeys.LOCAL_VARIABLES, scope="mIoU")
        running_px_acc_vars = tf.get_collection(tf.GraphKeys.LOCAL_VARIABLES, scope="px_acc")

        running_metrics_init = tf.group(tf.variables_initializer(var_list=running_miou_vars),
                                        tf.variables_initializer(var_list=running_px_acc_vars))

        # Create the training and summary operations
        trainable_var = tf.trainable_variables()
        train_op = net.add_train_ops(loss, trainable_var, global_step)

        logging.info("Setting up summary op...")
        summary_op = tf.summary.merge_all()
        summary_writer = tf.summary.FileWriter(FLAGS.logs_dir, net.sess.graph)

        # We assume a single records for train, val, test, but the
        # build_tfrecords_batch method can accept arbitrary paths
        path_to_train_tfrecord = os.path.join(
            FLAGS.path_to_records, "{}_train.tfrecords".format(FLAGS.records_prefix))
        path_to_val_tfrecord = os.path.join(
            FLAGS.path_to_records, "{}_val.tfrecords".format(FLAGS.records_prefix))
        path_to_test_tfrecord = os.path.join(
            FLAGS.path_to_records, "{}_test.tfrecords".format(FLAGS.records_prefix))

        if FLAGS.mode in ["train", "visualize"]:
            # Create data generators for train and validation
            image_batch, label_batch = build_tfrecords_batch(
                path_to_train_tfrecord, input_shape=SEMSEG_SHAPE, batch_size=FLAGS.batch_size,
                min_after_dequeue=FLAGS.min_after_dequeue, max_iterations=MAX_ITERATIONS)
            val_image_batch, val_label_batch = build_tfrecords_batch(
                path_to_val_tfrecord, input_shape=SEMSEG_SHAPE, batch_size=FLAGS.batch_size,
                min_after_dequeue=FLAGS.min_after_dequeue, max_iterations=MAX_ITERATIONS)

        # Re-run global variable initializer for the training ops just added and tfrecords
        init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
        net.sess.run(init_op)

        # We need to reload the model since we just wiped all variables by reiniting
        net.load_network_weights(net.network_weights_loc)

        if FLAGS.mode == "train":
            coord = tf.train.Coordinator()
            threads = tf.train.start_queue_runners(sess=net.sess, coord=coord)

            # Begin training
            for i in range(MAX_ITERATIONS):
                itr = tf.train.global_step(net.sess, global_step)

                train_images, train_annotations = net.sess.run([image_batch, label_batch])

                feed_dict = {image: train_images, annotation: train_annotations,
                             keep_prob: 0.85, is_training: True}

                net.sess.run(train_op, feed_dict=feed_dict)

                if itr % 10 == 0 and itr > 0:
                    train_loss, summary_str = net.sess.run([loss, summary_op], feed_dict=feed_dict)

                    logging.info("Step: %d, Train_loss:%g" % (itr, train_loss))
                    summary_writer.add_summary(summary_str, itr)

                if itr % 500 == 0 and itr > 0:
                    num_batches_per_epoch = (FLAGS.num_samples_per_val // FLAGS.batch_size) + 1
                    val_losses = []

                    # Reset mIoU running vars so we get a fresh evaluation for each validation
                    net.sess.run(running_metrics_init)

                    for i in range(num_batches_per_epoch):

                        val_images, val_annotations = net.sess.run([val_image_batch, val_label_batch])
                        val_feed_dict = {image: val_images, annotation: val_annotations,
                                         keep_prob: 1.0, is_training: False}

                        val_loss, _ = net.sess.run([loss, metrics_update_op], feed_dict=val_feed_dict)
                        val_losses.append(val_loss)

                    avg_val_loss = np.mean(val_losses)
                    best_loss_val = net.sess.run(best_loss)

                    # Get current and best mean IoUs and pixel accuracies
                    val_iou, best_iou_val = net.sess.run([mIoU, best_iou])
                    val_px_acc, best_px_acc_val = net.sess.run([px_acc, best_px_acc])

                    logging.info(
                        "{} ---> Validation_loss: {} (best: {}), Validation_IoU: {} (best: {}), "
                        "Validation pixel accuracy: {} (best: {})".format(
                                    datetime.datetime.now(),
                                    avg_val_loss, best_loss_val,
                                    val_iou, best_iou_val,
                                    val_px_acc, best_px_acc_val))

                    # If this is the best we've done on validation, update checkpoint
                    if avg_val_loss < best_loss_val:
                        net.sess.run(best_loss.assign(avg_val_loss))
                        net.sess.run(best_iou.assign(val_iou))
                        net.sess.run(best_px_acc.assign(val_px_acc))
                        logging.info(
                            "New best loss: {} with iou: {} and px acc: {} at step {}".format(
                                avg_val_loss, val_iou, val_px_acc, itr))

                        net.saver.save(net.sess,
                                       os.path.join(FLAGS.logs_dir, "best_model.ckpt"),
                                       global_step=itr)

                if itr >= MAX_ITERATIONS:
                    break

            coord.request_stop()
            coord.join(threads)

        elif FLAGS.mode == "test":
            if FLAGS.dataset == "ade20k":
                # We don't have ade20k test data, so test on validation set
                test_generator = read_and_decode_semseg_tfrecord(path_to_val_tfrecord)
            else:
                test_generator = read_and_decode_semseg_tfrecord(path_to_test_tfrecord)

            all_losses = []
            for X, Y in tqdm(test_generator, desc="Testing"):
                X, Y = np.expand_dims(X, axis=0), np.expand_dims(Y, axis=0)

                feed_dict = {image: X, annotation: Y, keep_prob: 1.0, is_training: False}
                test_loss, _ = net.sess.run([loss, update_op], feed_dict=feed_dict)

                all_losses.append(test_loss)

            test_iou = net.sess.run(mIoU)
            logging.info("Test Loss: {}, mIoU: {}".format(np.mean(all_losses), test_iou))

if __name__ == "__main__":
    main()


text:
################################################################################
# Module: setup_sp.py
# Description: this module contains functions to set up sample points stats within study regions

################################################################################

import osmnx as ox
import networkx as nx
import geopandas as gpd
import pandas as pd
import pandana as pdna
import numpy as np
from scipy.stats import zscore
import gc
import os
import csv

def read_proj_graphml(proj_graphml_filepath, ori_graphml_filepath, to_crs):
    """
    Read a projected graph from local disk if exist,
    otherwise, reproject origional graphml to the UTM zone appropriate for its geographic location,
    and save the projected graph to local disk

    Parameters
    ----------
    proj_graphml_filepath: string
        the projected graphml filepath
    ori_graphml_filepath: string
        the original graphml filepath
    to_crs: dict or string or pyproj.CRS
        project to this CRS

    Returns
    -------
    networkx multidigraph
    """
    # if the projected graphml file already exist in disk, then load it from the path
    if os.path.isfile(proj_graphml_filepath):
        print('Read network from disk.')
        return ox.load_graphml(proj_graphml_filepath)

    # else, read original study region graphml and reproject it
    else:
        print('Reproject network, and save the projected network to disk')

        # load and project origional graphml from disk
        G = ox.load_graphml(ori_graphml_filepath)
        G_proj = ox.project_graph(G, to_crs=to_crs)
        # save projected graphml to disk
        ox.save_graphml(G_proj, proj_graphml_filepath)

        return G_proj

def calc_sp_pop_intect_density_multi(G_proj, hexes, length, rows, node, index):
    """
    Calculate population and intersection density for each sample point

    This function is for multiprocessing.
    A subnetwork will be created for each sample point as a neighborhood and then intersect the hexes
    with pop and intersection data. Population and intersection density for each sample point are caculated by averaging the intersected hexes density data

    Parameters
    ----------
    G_proj: networkx multidigraph
    hexes: GeoDataFrame
        hexagon layers containing pop and intersection info
    length: int
        distance to search around the place geometry, in meters
    rows: int
        the number of rows to loop
    node: list
        the list of osmid of nodes
    index: int
        loop number

    Returns
    -------
    list
    """
    if index % 100 == 0:
        print('{0} / {1}'.format(index, rows))

    # create subgraph of neighbors centered at a node within a given radius.
    subgraph_proj = nx.ego_graph(G_proj,
                                 node,
                                 radius=length,
                                 distance='length')
    # convert subgraph into edge GeoDataFrame
    subgraph_gdf = ox.graph_to_gdfs(subgraph_proj,
                                    nodes=False,
                                    edges=True,
                                    fill_edge_geometry=True)

    # intersect GeoDataFrame with hexes
    if len(subgraph_gdf) > 0:
        intersections = gpd.sjoin(hexes,
                                  subgraph_gdf,
                                  how='inner',
                                  op='intersects')

        # drop all rows where 'index_right' is nan
        intersections = intersections[intersections['index_right'].notnull()]
        # remove rows where 'index' is duplicate
        intersections = intersections.drop_duplicates(subset=['index'])
        # return list of nodes with osmid, pop and intersection density
        return [
            node,
            float(intersections['pop_per_sqkm'].mean()),
            float(intersections['intersections_per_sqkm'].mean())
        ]
    else:
        return [node]

def calc_sp_pop_intect_density(osmid, G_proj, hexes, field_pop, field_intersection,
                      length, counter, rows):
    """
    Calculate population and intersection density for a sample point

    This function is apply the single thred method.
    A subnetwork will be created for each sample point as a neighborhood and then intersect the hexes
    with pop and intersection data. Population and intersection density for each sample point are caculated by averaging the intersected hexes density data

    Parameters
    ----------
    osmid: int
        the id of a node
    G_proj: networkx multidigraph
    hexes: GeoDataFrame
        hexagon layers containing pop and intersection info
    field_pop: str
        the field name of pop density
    field_intersection: str
        the field name of intersection density
    length: int
        distance to search around the place geometry, in meters
    counter: value
        counter for process times(Object from multiprocessing)
    rows: int
        the number of rows to loop

    Returns
    -------
    Series
    """
    # apply calc_sp_pop_intect_density_single function to get population and intersection density for sample point
    pop_per_sqkm, int_per_sqkm = calc_sp_pop_intect_density_single(osmid, G_proj, hexes,
                                                    length, counter, rows)
    return pd.Series({
        field_pop: pop_per_sqkm,
        field_intersection: int_per_sqkm
    })

def calc_sp_pop_intect_density_single(osmid, G_proj, hexes, length, counter, rows):
    """
    Calculate population and intersection density for a sample point

    This function is for single thred.
    A subnetwork will be created for each sample point as a neighborhood and then intersect the hexes
    with pop and intersection data. Population and intersection density for each sample point are caculated by averaging the intersected hexes density data

    Parameters
    ----------
    osmid: int
        the id of a node
    G_proj: networkx multidigraph
    hexes: GeoDataFrame
        hexagon layers containing pop and intersection info
    length: int
        distance to search around the place geometry, in meters
    counter: value
        counter for process times (object from multiprocessing)
    rows: int
        the number of rows to loop

    Returns
    -------
    tuple, (pop density, intersection density)
    """
    with counter.get_lock():
        counter.value += 1
        if counter.value % 100 == 0:
            print('{0} / {1}'.format(counter.value, rows))
    orig_node = osmid
    # create subgraph of neighbors centered at a node within a given radius.
    subgraph_proj = nx.ego_graph(G_proj,
                                 orig_node,
                                 radius=length,
                                 distance='length')
    # convert subgraph into edge GeoDataFrame
    subgraph_gdf = ox.graph_to_gdfs(subgraph_proj,
                                    nodes=False,
                                    edges=True,
                                    fill_edge_geometry=True)
    # intersect sample point GeoDataFrame with hexes
    if len(subgraph_gdf) > 0:
        intersections = gpd.sjoin(hexes,
                                  subgraph_gdf,
                                  how='inner',
                                  op='intersects')
        # drop all rows where 'index_right' is nan
        intersections = intersections[intersections['index_right'].notnull()]
        # remove rows where 'index' is duplicate
        intersections = intersections.drop_duplicates(subset=['index'])
        # return tuple, pop and intersection density for sample point
        return (intersections['pop_per_sqkm'].mean(),
                intersections['intersections_per_sqkm'].mean())
    else:
        return (np.nan, np.nan)

def createHexid(sp, hex):
    """
    Create hex_id for sample point, if it not exists

    Parameters
    ----------
    sp: GeoDataFrame
        sample point GeoDataFrame
    hex: GeoDataFrame
        hexagon GeoDataFrame

    Returns
    -------
    GeoDataFrame
    """
    if 'hex_id' not in sp.columns.tolist():
        # get sample point dataframe columns
        print('Create hex_id for sample points')
        samplePoint_column = sp.columns.tolist()
        samplePoint_column.append('index')

        # join id from hex to each sample point
        samplePointsData = gpd.sjoin(sp, hex, how='left', op='within')
        samplePointsData = samplePointsData[samplePoint_column].copy()
        samplePointsData.rename(columns={'index': 'hex_id'}, inplace=True)
        return samplePointsData
    else:
        print("hex_id' already in sample point.")

def create_pdna_net(gdf_nodes, gdf_edges, predistance=500):
    """
    Create pandana network to prepare for calculating the accessibility to destinations
    The network is comprised of a set of nodes and edges.

    Parameters
    ----------
    gdf_nodes: GeoDataFrame
    gdf_edges: GeoDataFrame
    predistance: int
        the distance of search (in meters), default is 500 meters

    Returns
    -------
    pandana network
    """
    # Defines the x attribute for nodes in the network
    gdf_nodes['x'] = gdf_nodes['geometry'].apply(lambda x: x.x)
    # Defines the y attribute for nodes in the network (e.g. latitude)
    gdf_nodes['y'] = gdf_nodes['geometry'].apply(lambda x: x.y)
    # Defines the node id that begins an edge
    gdf_edges['from'] = gdf_edges['u'].astype(np.int64)
    # Defines the node id that ends an edge
    gdf_edges['to'] = gdf_edges['v'].astype(np.int64)
    # Define the distance based on OpenStreetMap edges
    gdf_edges['length'] = gdf_edges['length'].astype(float)

    gdf_nodes['id'] = gdf_nodes['osmid'].astype(np.int64)
    gdf_nodes.set_index('id', inplace=True, drop=False)
    # Create the transportation network in the city
    # Typical data would be distance based from OSM or travel time from GTFS transit data
    net = pdna.Network(gdf_nodes['x'], gdf_nodes['y'], gdf_edges['from'],
                       gdf_edges['to'], gdf_edges[['length']])
    # Precomputes the range queries (the reachable nodes within this maximum distance)
    # so that aggregations don’t perform the network queries unnecessarily
    net.precompute(predistance + 10)
    return net

def cal_dist_node_to_nearest_pois(gdf_poi, distance, network, *args, filterattr=True):
    """
    Calculate the distance from each node to the first nearest destination
    within a given maximum search distance threshold
    If the nearest destination is not within the distance threshold, then it will be coded as -999

    Parameters
    ----------
    gdf_poi: GeoDataFrame
        GeoDataFrame of destination point-of-interest
    distance: int
        the maximum search distance
    network: pandana network
    args* : arguments
        list of field names of destination categories
    filterattr: bool
        indicate to public open space or not (default: {True})
        if False indicates to process public open space layer

    Returns
    -------
    GeoDataFrame
    """
    gdf_poi['x'] = gdf_poi['geometry'].apply(lambda x: x.x)
    gdf_poi['y'] = gdf_poi['geometry'].apply(lambda x: x.y)
    # if True, process the give open space layer
    if filterattr is True:
        appended_data = []
        # iterate over each destination category
        for x in args:
            # initialize the destination point-of-interest category
            # the positions are specified by the x and y columns (which are Pandas Series)
            # at a max search distance for up to the first nearest points-of-interest
            network.set_pois(x[0], distance, 1,
                             gdf_poi[gdf_poi['dest_name_full'] == x[0]]['x'],
                             gdf_poi[gdf_poi['dest_name_full'] == x[0]]['y'])
            # return the distance to the first nearest destination category
            # if zero destination is within the max search distance, then coded as -999
            dist = network.nearest_pois(distance, x[0], 1, -999)

            # important to convert columns index type to str
            dist.columns = dist.columns.astype(str)
            # change the index name corresponding to each destination name
            columnName = x[1]
            dist.rename(columns={'1': columnName}, inplace=True)
            appended_data.append(dist)
        # return a GeoDataFrame with distance to the nearest destination from each source node
        gdf_poi_dist = pd.concat(appended_data, axis=1)
        return gdf_poi_dist
    # if False, process the public open space layer
    else:
        for x in args:
            network.set_pois(x[0], distance, 1, gdf_poi['x'], gdf_poi['y'])
            dist = network.nearest_pois(distance, x[0], 1, -999)
            dist.columns = dist.columns.astype(str)
            columnName = x[1]
            dist.rename(columns={'1': columnName}, inplace=True)
            return dist

def convert_dist_to_binary(gdf, *columnNames):
    """
    Convert numerical distance to binary, 0 or 1

    Parameters
    ----------
    gdf: GeoDataFrame
        GeoDataFrame with distance between nodes and nearest destination
    *columnNames: list
        list of column names of original dist columns and new binary columns
        eg. [[nearest_node_pos_dist], [nearest_node_pos_dist_binary]]

    Returns
    -------
    GeoDataFrame
    """
    for x in columnNames:
        #specify original column names with distance, and new binary column name
        columnName = x[0]
        columnBinary = x[1]
        # replace ditance value to 0 if the disntace is coded as -999
        # indicating that the nearest destination is not within the max search distance
        # otherwise,  replace ditance value to 0
        gdf[columnBinary] = np.where(gdf[columnName] == -999, 0, 1)
    return gdf

def cal_zscores(gdf, oriFieldNames, newFieldNames):
    """
    Claculate z-scores for variables

    Parameters
    ----------
    gdf: GeoDataFrame
    orifieldNames: list
        list contains origional field names of the columns needed to calculate zscores,
    newfieldNames: list
        list contains new field name after calculate the zscores

    Returns
    -------
    GeoDataFrame
    """
    #zip the old and new field names together
    fieldNames = list(zip(oriFieldNames, newFieldNames))
    for fields in fieldNames:
        #specify original field needed to calculate zscores, and the new field name after zscores
        orifield, newfield = fields[0], fields[1]
        #caluclate zscores within the GeoDataFrame
        gdf[newfield] = gdf[[orifield]].apply(zscore)
    return gdf

def split_list(alist, wanted_parts=1):
    """
    split list

    Parameters
    ----------
    alist: list
        the split list
    wanted_parts: int
        the number of parts (default: {1})

    Returns
    -------
    list
    """
    length = len(alist)
    #return all parts in a list, like [[],[],[]]
    return [
        alist[i * length // wanted_parts:(i + 1) * length // wanted_parts]
        for i in range(wanted_parts)
    ]

max_stars_repo_name: thuanguy/global-indicators

max_issues_repo_name: thuanguy/global-indicators

max_forks_repo_name: thuanguy/global-indicators

reason: import numpy,from scipy,import networkx


text:
"""
Test bonn data using previously generated model.
"""
from argparse import ArgumentParser

import numpy as np
from sklearn.preprocessing import LabelBinarizer
from sklearn.manifold import TSNE
from keras import layers, regularizers, models

from flowcat import classifier, utils, io_functions
from flowcat.constants import DEFAULT_CLASSIFIER_CONFIG, GROUPS, DEFAULT_CLASSIFIER_ARGS
from flowcat import flowcat_api as fc_api
from flowcat.classifier import som_dataset
from flowcat.classifier.models import create_model_multi_input

def load_model(path):
    binarizer = io_functions.load_joblib(path / "binarizer.joblib")
    model = models.load_model(str(path / "model.h5"))
    return binarizer, model

def main(args):
    dataset = som_dataset.SOMDataset.from_path(args.input)
    val = args.val
    train = args.train
    OUTPUT = args.output

    groups = ["MCL", "PL"]
    tubes = ("1")
    mapping = None
    balance = {
        "MCL": 20,
        "PL": 20,
    }

    config = classifier.SOMClassifierConfig(**{"tubes": {tube: dataset.config[tube] for tube in tubes},
                                               "groups": groups,
                                               "pad_width": 2,
                                               "mapping": mapping,
                                               "cost_matrix": None,
                                               })
    val = io_functions.load_json(val)
    validate_dataset = dataset.filter(labels=val)

    labels = io_functions.load_json(train)
    train_dataset = dataset.filter(labels=labels)

    train_dataset, validate_dataset = fc_api.prepare_classifier_train_dataset(train_dataset,split_ratio=0.9,
                                                                              groups=groups,
                                                                              mapping=mapping,
                                                                              balance=balance, val_dataset = validate_dataset)

    print(train_dataset.group_count)
    print(validate_dataset.group_count)

    binarizer, model = load_model(args.model)

    trainseq = som_dataset.SOMSequence(train_dataset, binarizer, tube=config.tubes, pad_width=config.pad_width)
    validseq = som_dataset.SOMSequence(validate_dataset, binarizer, tube=config.tubes, pad_width=config.pad_width)

    model.fit_generator(
        generator=trainseq,
        epochs=10,
        validation_data=validseq)

    args.output.mkdir(parents=True, exist_ok=True)
    io_functions.save_joblib(binarizer, OUTPUT / "binarizer.joblib")
    model.save(str(args.output / "model.h5"))

    io_functions.save_json(config.to_json(), OUTPUT / "config.json")
    io_functions.save_json(validseq.dataset.labels, OUTPUT / "ids_validate.json")
    io_functions.save_json(trainseq.dataset.labels, OUTPUT / "ids_train.json")

if __name__ == "__main__":
    PARSER = ArgumentParser()
    PARSER.add_argument("--input", type=utils.URLPath)
    PARSER.add_argument("--output", type=utils.URLPath)
    PARSER.add_argument("--val", type=utils.URLPath)
    PARSER.add_argument("--train", type=utils.URLPath)
    PARSER.add_argument("model", type=utils.URLPath)
    main(PARSER.parse_args())

max_stars_repo_name: NandithaMallesh/mergeTL

max_issues_repo_name: NandithaMallesh/mergeTL

max_forks_repo_name: NandithaMallesh/mergeTL


text:
import sys
from plot_graph.plot_graph import PlotGraph

from sympy import *
import sympy as sp

import gi
gi.require_version('Gtk', '3.0')
from gi.repository import GLib, Gio, Gtk

from matplotlib.backends.backend_gtk3agg import (
    FigureCanvasGTK3Agg as FigureCanvas)
from matplotlib.figure import Figure

from matplotlib.backends.backend_gtk3 import (
    NavigationToolbar2GTK3 as NavigationToolbar)
import numpy as np

class MainWindow(Gtk.Window):

    def __init__(self):
        Gtk.Window.__init__(self, title="Numeric methods")
        self.set_border_width(10)
        self.set_size_request(872, 657)
        #self.set_resizable(resizable=False)
        self.pl = PlotGraph()

        # Layout grid
        self.grid = Gtk.Grid(column_homogeneous=True, column_spacing=10, row_spacing=10)
        self.add(self.grid)

        # All entries (equation, x0, y0, step, bound)
        # equation
        self.equation = Gtk.Entry(placeholder_text="write your equation")
        self.equation.set_size_request(357, 47)

        # x0
        self.x_entry = Gtk.Entry(placeholder_text="initial point x")
        self.x_entry.set_size_request(134, 37)

        # y0
        self.y_entry = Gtk.Entry(placeholder_text="initial point y")
        self.y_entry.set_size_request(134, 37)

        # grid
        self.step = Gtk.Entry(placeholder_text="step")
        self.step.set_size_request(134, 37)

        # grid bound
        self.grid_bound = Gtk.Entry(placeholder_text="step bound")
        self.grid_bound.set_size_request(134, 37)

        # bound
        self.bound = Gtk.Entry(placeholder_text="x bound")
        self.bound.set_size_request(134, 37)

        # flag to check which error should be plotted(local or global)
        self.error_flag = False  # when False it means local, when True it means global

        # creating button
        self.button = Gtk.Button(label="PLOT GRAPH")
        self.button.connect("clicked", self.draw)
        self.button.set_size_request(115, 47)

        self.canvas = FigureCanvas(self.pl.fig)
        self.canvas.set_size_request(800, 600)

        # check buttons (for choice of methods that would be plotted)
        ex_box = Gtk.Box()
        label_ex = Gtk.Label("Exact")
        self.exact_check = Gtk.CheckButton()
        self.exact_check.connect("toggled", self.toggled)
        ex_box.pack_start(label_ex, True, True, 0)
        ex_box.pack_start(self.exact_check, True, True, 0)

        label_eul = Gtk.Label("Euler")
        eul_box = Gtk.Box()
        self.eul_check = Gtk.CheckButton()
        self.eul_check.connect("toggled", self.toggled)
        eul_box.pack_start(label_eul, True, True, 0)
        eul_box.pack_start(self.eul_check, True, True, 0)

        label_imp = Gtk.Label("Improved Eul")
        imp_box = Gtk.Box()
        self.imp_check = Gtk.CheckButton()
        self.imp_check.connect("toggled", self.toggled)
        imp_box.pack_start(label_imp, True, True, 0)
        imp_box.pack_start(self.imp_check, True, True, 0)

        label_kutta = Gtk.Label("Runge-Kutta  ")
        kutta_box = Gtk.Box()
        self.kutta_check = Gtk.CheckButton()
        self.kutta_check.connect("toggled", self.toggled)
        kutta_box.pack_start(label_kutta, True, True, 0)
        kutta_box.pack_start(self.kutta_check, True, True, 0)

        # global error canvas
        self.gl_canvas = FigureCanvas(self.pl.fig_gl_err)
        self.gl_canvas.set_size_request(800, 600)

        self.toolbar = NavigationToolbar(self.canvas, Gtk)
        self.toolbox = Gtk.Box()
        self.toolbox.add(self.toolbar)

        radio_box = Gtk.Box()
        loc_button = Gtk.RadioButton.new_with_label_from_widget(None, "Local")
        loc_button.connect("toggled", self.radio_toggled, "local", self)
        radio_box.pack_start(loc_button, False, False, 10)

        glob_button = Gtk.RadioButton.new_from_widget(loc_button)
        glob_button.set_label("Global")
        glob_button.connect("toggled", self.radio_toggled, "global", self)
        radio_box.pack_start(glob_button, False, False, 10)

        # Adding entries and buttons to grid
        self.grid.attach(self.equation, 2, 0, 3, 1)
        self.grid.attach(self.button, 4, 2, 1, 1)
        self.grid.attach(self.x_entry, 2, 1, 1, 1)
        self.grid.attach(self.y_entry, 3, 1, 1, 1)
        self.grid.attach(self.step, 2, 2, 1, 1)
        self.grid.attach(self.bound, 4, 1, 1, 1)
        self.grid.attach(self.toolbox, 2, 3, 1, 1)
        self.grid.attach(self.canvas, 2, 4, 3, 4)
        self.grid.attach(self.gl_canvas, 5, 4, 3, 4)
        self.grid.attach(ex_box, 5, 0, 1, 1)
        self.grid.attach(eul_box, 5, 1, 1, 1)
        self.grid.attach(imp_box, 7, 0, 1, 1)
        self.grid.attach(kutta_box, 7, 1, 1, 1)
        self.grid.attach(self.grid_bound, 3, 2, 1, 1)
        self.grid.attach(radio_box, 6, 2, 1, 1)

    @staticmethod
    def toggled(check_butt):
        if check_butt.get_active():
            print("Active")

    @staticmethod
    def radio_toggled(button, name, activity):
        if button.get_active() and name == "global":
            activity.error_flag = True
        else:
            activity.error_flag = False

    def draw(self, wid):
        # draw the graph on ui, click listener on button 'plot graph'
        self.grid.remove_row(5)

        if self.exact_check.get_active():
            self.pl.ex_flag = True
        else:
            self.pl.ex_flag = False

        if self.kutta_check.get_active():
            self.pl.kutta_flag = True
        else:
            self.pl.kutta_flag = False

        if self.eul_check.get_active():
            self.pl.eul_flag = True
        else:
            self.pl.eul_flag = False

        if self.imp_check.get_active():
            self.pl.imp_flag = True
        else:
            self.pl.imp_flag = False

        # check if method is chosen
        if not self.pl.imp_flag and not self.pl.kutta_flag and not self.pl.eul_flag:
            dialog = Gtk.MessageDialog(self, 0, Gtk.MessageType.INFO,
                                       Gtk.ButtonsType.OK, "You should choose at least one method")
            dialog.format_secondary_text(
                "No methods are chosen")
            dialog.run()
            dialog.destroy()
            return

        # user input (some bad code style, need to create function for checking)
        try:
            x0 = float(self.x_entry.get_text())
        except ValueError:
            dialog = Gtk.MessageDialog(self, 0, Gtk.MessageType.INFO,
                                       Gtk.ButtonsType.OK, "Wrong initial x")
            dialog.format_secondary_text(
                "Your initial x is not a number")
            dialog.run()
            dialog.destroy()
            return

        try:
            y0 = float(self.y_entry.get_text())
        except ValueError:
            dialog = Gtk.MessageDialog(self, 0, Gtk.MessageType.INFO,
                                       Gtk.ButtonsType.OK, "Wrong initial y")
            dialog.format_secondary_text(
                "Your initial y is not a number")
            dialog.run()
            dialog.destroy()
            return

        try:
            bound = float(self.bound.get_text())
        except ValueError:
            dialog = Gtk.MessageDialog(self, 0, Gtk.MessageType.INFO,
                                       Gtk.ButtonsType.OK, "Wrong x bound")
            dialog.format_secondary_text(
                "Your x bound is not a number")
            dialog.run()
            dialog.destroy()
            return

        try:
            step = float(self.step.get_text())
        except ValueError:
            dialog = Gtk.MessageDialog(self, 0, Gtk.MessageType.INFO,
                                       Gtk.ButtonsType.OK, "Wrong step")
            dialog.format_secondary_text(
                "Your step is not a number")
            dialog.run()
            dialog.destroy()
            return

        try:
            step_bound = float(self.grid_bound.get_text())
        except ValueError:
            dialog = Gtk.MessageDialog(self, 0, Gtk.MessageType.INFO,
                                       Gtk.ButtonsType.OK, "Wrong step bound")
            dialog.format_secondary_text(
                "Your step bound is not a number")
            dialog.run()
            dialog.destroy()
            return

        equation = str(self.equation.get_text())

        self.pl.init_methods(x0, y0, bound, step, equation)
        self.pl.plot_graph()
        self.canvas = FigureCanvas(self.pl.fig)
        self.canvas.show()

        self.toolbar.canvas = self.canvas
        self.toolbar.show()

        if self.error_flag:
            self.pl.init_global(x0, y0, bound, step, step_bound, equation)
            self.pl.plot_global()
            self.gl_canvas = FigureCanvas(self.pl.fig_global)
            self.gl_canvas.show()
        else:
            self.pl.init_gl_error()
            self.pl.plot_gll_err()
            self.gl_canvas = FigureCanvas(self.pl.fig_gl_err)
            self.gl_canvas.show()

        self.grid.attach(self.canvas, 2, 4, 3, 4)
        self.grid.attach(self.gl_canvas, 5, 4, 3, 4)
        self.show_all()

window = MainWindow()
window.connect("delete-event", Gtk.main_quit)
window.show_all()
Gtk.main()

reason: import numpy,import sympy,from sympy


text:
import sys, os
import numpy as np
import glob
from matplotlib import pylab as plt
from numpy.linalg import svd
from PIL import Image
import time

class TRPCA:

    def converged(self, L, E, X, L_new, E_new):
        eps = 1e-8
        condition1 = np.max(L_new - L) < eps
        condition2 = np.max(E_new - E) < eps
        condition3 = np.max(L_new + E_new - X) < eps
        return condition1 and condition2 and condition3

    def SoftShrink(self, X, tau):
        z = np.sign(X) * (abs(X) - tau) * (np.sign(abs(X) - tau) + 1)/ 2
        return z

    def SVDShrink(self, X, tau):
        u, s, v = svd(X, full_matrices = False)
        s_bar = self.SoftShrink(s, tau)
        return np.dot(np.dot(u, np.diag(s_bar)), v)

    def ADMM(self, X):
        m, n = X.shape
        rho = 1.5
        mu = 1e-3
        mu_max = 1e10
        max_iters = 1000
        lamb = 1/np.sqrt(max(m, n))
        L = np.zeros((m, n), float)
        E = np.zeros((m, n), float)
        Y = np.zeros((m, n), float)
        iters = 0
        while True:
            iters += 1
            L_new = self.SVDShrink(X - E - (1/mu) * Y, 1/mu)
            E_new = self.SoftShrink(X - L_new - (1/mu) * Y, lamb/mu)
            Y += mu * (L_new + E_new - X)
            mu = min(rho * mu, mu_max)
            if self.converged(L, E, X, L_new, E_new) or iters >= max_iters:
                return L_new, E_new
            else:
                L, E = L_new, E_new
                print(np.max(X - L - E))

# for image denoising
# Load input data as gray image
# set the file path of your photo which you want to denoise
X = np.array(Image.open('set your path').convert('L'))
TRPCA = TRPCA()
L, S = TRPCA.ADMM(X)
plt.subplot(131)
plt.imshow(L)
plt.gray()
plt.subplot(132)
plt.imshow(S)
plt.gray()
plt.subplot(133)
plt.imshow(X)
plt.gray()
plt.show()

# for background monitoring
# files = glob.glob('set your directory/*.jpg')
# matrix = []
# for file in files:
#    img = Image.open(file).convert('L')
#    h,w=np.array(img).shape
#    pixels = list(img.getdata())
#    matrix.append(pixels)
# X=(np.array(matrix).astype(np.float64))
# TRPCA = TRPCA()
# L, S = TRPCA.ADMM(X)
# L = L[:,0].reshape((h,w))
# S = S[:,0].reshape((h,w))