text:
import nltk
import pandas as pd
import numpy as np
import re
import codecs
import json

import re
import numpy as np
import pandas as pd
from pprint import pprint
import pickle
# Gensim
import gensim
import gensim.corpora as corpora
from gensim.utils import simple_preprocess
from gensim.models import CoherenceModel
from gensim.models import Phrases
from gensim.corpora import Dictionary, MmCorpus
from gensim.models.word2vec import LineSentence
from gensim.models.ldamulticore import LdaMulticore
# spacy for lemmatization
import spacy
# NLTK for text cleaning
from nltk.tokenize import sent_tokenize, word_tokenize, RegexpTokenizer
from nltk.classify import NaiveBayesClassifier
from nltk.corpus import stopwords, names
from nltk.tokenize import RegexpTokenizer
from nltk import tokenize
from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer
from nltk.sentiment.vader import SentimentIntensityAnalyzer
# TextBlob package for translation and spelling correction
from textblob import TextBlob

nlp = spacy.load('en')
# Plotting tools
import pyLDAvis
import pyLDAvis.gensim  # don't skip this
import matplotlib.pyplot as plt
# %matplotlib inline

# Enable logging for gensim - optional
import logging
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.ERROR)

import warnings
warnings.filterwarnings("ignore",category=DeprecationWarning)

### Step 1: Initialization.

# Set up tokenizer.
tokenizer = RegexpTokenizer(r'\w+')
# Set up stop words.
en_stop = set(stopwords.words('english'))
# Set up stemmer.
p_stemmer = PorterStemmer()
# For sentiment analysis
senti = SentimentIntensityAnalyzer()

# Load saved models.
# I trained three LDA models from step 5, they have slightly different topics.
# I trained three models because there is randomness in LDA training.
ldamodel1 = pickle.load(open("PATH/model/lda_1.pickle", "rb"))
dict1 = pickle.load(open("PATH/model/dictionary_1.pickle", "rb"))
corpus1 = pickle.load(open("PATH/model/corpus_1.pickle", "rb"))

# # Topic dictionary
# lda_topics={
#                 0:'Parking Service',
#                 1:'Sleep Quality',
#                 2:'WiFi',
#                 3:'Online Service',
#                 4:'Overall Experience',
#                 5:'Value for Money',
#                 6:'Swimming Pool/Spa',
#                 7:'Front Desk Service',
#                 8:'Food', ''
#                 9:'Cleanliness',
#                 10:'Surroundings',
#                 11:'Distance from Transportation',
#                 12:'Booking Experience',
#                 13:'Hotel Staff',
#                 14:'Early Check-in/Late Check-out',
#                 15:'Noise'
#             }

mallet_lda_topics={
                0:'Hotel Staff',
                1:'Accessibility',
                2:'Food',
                3:'Overall Experience',
                4:'Noise',
                5:'Value for Money',
                6:'Room Amenities',
                7:'Location in the city',
                8:'Overall Experience',
                9:'Cleanliness',
                10:'Early Check-in/Late Check-out',
                11:'Health and Wellness Amenities',
                12:'Booking Experience',
                13:'Sleep Quality',
                14:'Parking Facility'
            }

# helper functions for text preprocessing & LDA modeling:

def punct_space(token):
    """
    helper function to eliminate tokens
    that are pure punctuation or whitespace
    """

    return token.is_punct or token.is_space or token.like_num or token.is_digit

def line_review(filename):
    """
    generator function to read in reviews from Pandas Series
    and un-escape the original line breaks in the text
    """

    #with codecs.open(filename, encoding='utf_8') as f:
    for review in filename:
        yield review.replace('\\n', '\n')

def lemmatized_sentence_corpus(filename):
    """
    generator function to use spaCy to parse reviews,
    lemmatize the text, and yield sentences
    """

    for parsed_review in nlp.pipe(line_review(filename), batch_size=10000, n_threads=10):
        for sent in parsed_review.sents:
            yield u' '.join([token.lemma_ for token in sent
                             if not punct_space(token)])

def trigram_bow_generator(filepath):
    """
    generator function to read reviews from a file
    and yield a bag-of-words representation
    """
    # load finished dictionary from disk
    trigram_dictionary = Dictionary.load('./models2/trigram_dict_all.dict')

    for review in LineSentence(filepath):
        yield trigram_dictionary.doc2bow(review)

def LDA_Review(review_df,min_topic_freq=0):
    """
    Takes Pandas series as input,
    consisting of one review as
    text string per row
    """
    from tqdm import tqdm
    """
    accept the original text of a review and (1) parse it with spaCy,
    (2) apply text pre-proccessing steps, (3) create a bag-of-words
    representation, (4) create an LDA representation, and
    (5) print a sorted list of the top topics in the LDA representation
    """
    text = review_df['FullReview']
    # parse the review text with spaCy
    with codecs.open('./uni_temporary.txt', 'w', encoding='utf_8') as f:
        for sentence in tqdm(lemmatized_sentence_corpus(text)):
            # print(sentence)
            f.write(sentence + '\n')
    f.close()
    # load and apply the first-order and secord-order phrase models
    bigram_model = Phrases.load('./models2/bigram_model.txt')
    trigram_model = Phrases.load('./models2/trigram_model.txt')

    unigram_review = LineSentence('./uni_temporary.txt')
    bigram_review = bigram_model[unigram_review]
    trigram_review = trigram_model[bigram_review]
    # remove any remaining stopwords
    trigram_review = [term for term in trigram_review
                        if term not in spacy.lang.en.stop_words.STOP_WORDS]
    with codecs.open('./tri_temporary.txt', 'w', encoding='utf_8') as ftri:
        for sentence in trigram_review:
            sentence = u' '.join(sentence)
            ftri.write(sentence + '\n')
    ftri.close()

    trigram_dictionary = Dictionary.load('./models2/trigram_dict_all.dict')
    lda = LdaMulticore.load('./models2/lda_model')
    trigram_review = LineSentence('./tri_temporary.txt')
    # create a bag-of-words representation
    review_bow = trigram_dictionary.doc2bow(trigram_review)
    # create an LDA representation
    review_lda = lda.get_document_topics(review_bow)
    review_lda = sorted(review_lda, key=itemgetter(1),reverse=True)
    for topic_number, freq in review_lda:
        if freq < min_topic_freq:
            break
        # print the most highly related topic names and frequencies
        print('{:25} {}'.format(lda_topics[topic_number], round(freq, 3)))

    ### Step 2: Generate the contents of the doctors' snapshots.

    counter = 0
    # The temporary string that stores all of the review highlights in each round of the for loop below.
    big_str = []
    # For every doctor, find two things:
    #     1. The most mentioned FIVE topics in their reviews.
    #         1.1 The sentiments of these topics.
    #     2. The 3 most positive sentences and the 3 most negative sentences.
    #         2.1 Rank all sentences according to sentiment analysis.
    # I do NOT keep info about individual reviews. All sentences are stored in a
    # long list regardless of whether they are from the same reviews or not!
    ###########################################################################
    # Build sentence dataframe for the current doctor.
    ###########################################################################
    this_hotel = pd.DataFrame(columns = ["HotelName","Sentence", "Sentiment_neg",
                            "Sentiment_neu","Sentiment_pos",
                            "Sentiment_compound", "topic_1", "topic1_score",
                            "topic_2", "topic2_score"])
    sent_count = 0

    # For every review sentence
    for sentence in unigram_review:
        # Assess sentiment.
        sentiments = senti.polarity_scores(sentence)
        sentiment_neg = sentiments["neg"]
        sentiment_neu = sentiments["neu"]
        sentiment_pos = sentiments["pos"]
        sentiment_compound = sentiments["compound"]
        # Assign topic.
        # Default topic to -1.
        this_topic = -1
        # Preprocess sentence.
        sent_tokens = tokenizer.tokenize(str(sentence).lower())
        cleaned_sent = [p_stemmer.stem(i) for i in sent_tokens]
        # Evaluate for topic.
        sent_topics = []
        for mod_id in range(0, mod_num):
            model = ldamodel[mod_id]
            dicti = dictionary[mod_id]
            lda_score = model[dicti.doc2bow(cleaned_sent)]
            for item in lda_score:
                sent_topics.append((mod_id, item[0], item[1]))
        sent_topics =  sorted(sent_topics, key=lambda x: x[2], reverse=True)
        # Assign the most relevant topic to a sentence only if the topic is more than 70% dominant.
        if sent_topics[0][2] > 0.7:
            this_topic = topics_matrix[sent_topics[0][0]][sent_topics[0][1]]

        # Add procressed sentence and its meta information to the sentence dataframe.
        this_doc.loc[sent_count] = [sentence, sentiment, this_topic, sent_topics[0][2]]
        sent_count += 1

    ###########################################################################
    # Compiling results for a hotel.
    ###########################################################################
    # Review highlights.
    # Save the most positive and negative sentiments.
    this_doc2 = this_doc.sort_values(["sentiment"], ascending=[0]).reset_index(drop=True)
    this_doc2 = this_doc2.loc[this_doc2["topic"] != -1].reset_index(drop=True)
    this_doc2 = this_doc2.loc[this_doc2["topic_score"] > 0.5].reset_index(drop=True)
    sent_count_2 = len(this_doc2)
    composite = "NONE"
    # Save the most polarizing sentiments only if there are at least 6 sentences.
    if sent_count_2 > 5:
        sent1 = sent2 = sent3 = sent4 = sent5 = sent6 = ""
        # Only keep positive sentiment if its score is above 0.4 (within [-1, 1]).
        if this_doc2.loc[0]["sentiment"] > 0.4:
            sent1 = this_doc2.loc[0]["sentence"]
        if this_doc2.loc[1]["sentiment"] > 0.4:
            sent2 = this_doc2.loc[1]["sentence"]
        if this_doc2.loc[2]["sentiment"] > 0.4:
            sent3 = this_doc2.loc[2]["sentence"]
        # Only keep positive sentiment if its score is below -0.2 (within [-1, 1]).
        if this_doc2.loc[sent_count_2-1]["sentiment"] < -0.2:
            sent4 = this_doc2.loc[sent_count_2-1]["sentence"]
        if this_doc2.loc[sent_count_2-2]["sentiment"] < -0.2:
            sent5 = this_doc2.loc[sent_count_2-2]["sentence"]
        if this_doc2.loc[sent_count_2-3]["sentiment"] < -0.2:
            sent6 = this_doc2.loc[sent_count_2-3]["sentence"]
        composite = sent1 + "SSEEPP" + sent2 + "SSEEPP" + sent3 + "SSEEPP" + sent4 + "SSEEPP" + sent5 + "SSEEPP" + sent6 + "SSEEPP" + str(sent_count)
    # Add review highlights to the doctor dataframe.
    doctor_info.set_value(doctor_id, "summary", composite)

    # Top topics and their ratings.
    # Ratings are the percent positive sentences belonging to a topic.
    doc_topics = [ [ 0 for i in range(2) ] for j in range(topic_num) ]  # [total count, count positive]
    for index2 in range(0, len(this_doc2)):
        topic_index = this_doc2.loc[index2]["topic"]
        if topic_index != -1:
            doc_topics[topic_index][0] += 1
            topic_sentiment = this_doc2.loc[index2]["sentiment"]
            # A topic sentence if positive if its sentiment is bigger than 0.1.
            if topic_sentiment > 0.1:
                doc_topics[topic_index][1] += 1
    # Do not display dentist stuff for non-dentist
    if not is_dentist:
        doc_topics[3][0] = 0
    # Do not output "positive comment" as a topic. It is non-informative.
    doc_topics[0][0] = 0

    # Putting the results into a format to be sparsed by the webapp.
    doc_topic_tuples = []
    for index3, item in enumerate(doc_topics):
        doc_topic_tuples.append((index3, item[0], item[1]))
    doc_topic_tuples =  sorted(doc_topic_tuples, key=lambda x: x[1], reverse=True)
    for index4 in range(0, 5):
        if doc_topic_tuples[index4][1] >= 10:
            topic_name = topics[doc_topic_tuples[index4][0]][0]
            percent_positive = str(int(doc_topic_tuples[index4][2]/doc_topic_tuples[index4][1] * 100))
            composite = topic_name + "SSEEPP" + percent_positive + "SSEEPP" + str(doc_topic_tuples[index4][1])
            doctor_info.set_value(doctor_id, "percent{0}".format(str(index4+1)), composite)

            print(topic_name, "XXXXXX", doctor_info.loc[doctor_id]["specialty"])
            big_str.append(topic_name + "XXXXXX" + str(doctor_info.loc[doctor_id]["specialty"]))
        else:
            doctor_info.set_value(doctor_id, "percent{0}".format(str(index4+1)), "NONE")

    # Print progress.
    print(counter/5088)
    counter += doctor_review_count
    del this_doc
    del this_doc2

# Save the updated doctor dataframe containing snapshot information.
doctor_info.to_csv("PATH/result/all_doctors.csv")

max_stars_repo_name: Bhanuvadlamudi/smarthotels

max_issues_repo_name: Bhanuvadlamudi/smarthotels

max_forks_repo_name: Bhanuvadlamudi/smarthotels


text:
from rlpy.representations import OMPTD, IndependentDiscretization
from rlpy.domains import GridWorld
import numpy as np
from rlpy.tools import __rlpy_location__
import os

def test_bag_creation():
    """
    Ensure create appropriate # of conjunctions, that they have been
    instantiated properly, and there are no duplicates.
    """
    mapDir = os.path.join(__rlpy_location__, "domains", "GridWorldMaps")
    mapfile = os.path.join(mapDir, "4x5.txt")  # expect 4*5 = 20 states
    domain = GridWorld(mapfile=mapfile)

    initial_representation = IndependentDiscretization(domain)
    max_batch_discovery = np.inf
    batch_threshold = 1e-10
    discretization = 20
    bag_size = 100000  # We add all possible features

    rep = OMPTD(
        domain,
        initial_representation,
        discretization,
        max_batch_discovery,
        batch_threshold,
        bag_size,
        sparsify=False,
    )
    assert rep.total_feature_size == 9 + 20
    assert rep.features_num == 9

    # Compute full (including non-discovered) feature vec for a few states
    states = np.array([[0, 0], [0, 1], [1, 0], [1, 1]])
    rep.calculate_full_phi_normalized(states)
    phi_states = rep.fullphi
    phi_states[phi_states > 0] = True
    true_phi_s1 = np.zeros(len(phi_states[0, :]))
    true_phi_s1[0] = True
    true_phi_s1[4] = True  # TODO - could be [4] depending on axes, check.
    true_phi_s1[9] = True  # The conjunction of [0,0]
    assert np.all(true_phi_s1 == phi_states[0, :])  # expected feature vec returned
    assert sum(phi_states[0, :]) == 3  # 2 original basic feats and 1 conjunction

    true_phi_s2 = np.zeros(len(phi_states[0, :]))
    true_phi_s2[0] = True
    true_phi_s2[5] = True  # TODO - could be [4] depending on axes, check.
    true_phi_s2[10] = True  # The conjunction of [0,0]
    assert np.all(true_phi_s2 == phi_states[1, :])  # expected feature vec returned
    assert sum(phi_states[1, :]) == 3  # 2 original basic feats and 1 conjunction

    true_phi_s3 = np.zeros(len(phi_states[0, :]))
    true_phi_s3[1] = True
    true_phi_s3[4] = True  # TODO - could be [4] depending on axes, check.
    true_phi_s3[14] = True  # The conjunction of [0,0]
    assert np.all(true_phi_s3 == phi_states[2, :])  # expected feature vec returned
    assert sum(phi_states[2, :]) == 3  # 2 original basic feats and 1 conjunction

    true_phi_s4 = np.zeros(len(phi_states[0, :]))
    true_phi_s4[1] = True
    true_phi_s4[5] = True  # TODO - could be [4] depending on axes, check.
    true_phi_s4[15] = True  # The conjunction of [0,0]
    assert np.all(true_phi_s4 == phi_states[3, :])  # expected feature vec returned
    assert sum(phi_states[3, :]) == 3  # 2 original basic feats and 1 conjunction

def test_batch_discovery():
    """
    Test feature discovery from features available in bag, and that appropriate
    feats are activiated in later calls to phi_nonterminal()
    """
    mapDir = os.path.join(__rlpy_location__, "domains", "GridWorldMaps")
    mapfile = os.path.join(mapDir, "4x5.txt")  # expect 4*5 = 20 states
    domain = GridWorld(mapfile=mapfile)

    initial_representation = IndependentDiscretization(domain)
    max_batch_discovery = np.inf
    batch_threshold = 1e-10
    discretization = 20
    bag_size = 100000  # We add all possible features

    rep = OMPTD(
        domain,
        initial_representation,
        discretization,
        max_batch_discovery,
        batch_threshold,
        bag_size,
        sparsify=False,
    )
    states = np.array([[0, 0], [0, 2]])
    activePhi_s1 = rep.phi_non_terminal(states[0, :])
    activePhi_s2 = rep.phi_non_terminal(states[1, :])
    phiMatr = np.zeros((2, len(activePhi_s1)))
    phiMatr[0, :] = activePhi_s1
    phiMatr[1, :] = activePhi_s2
    td_errors = np.array([2, 5])
    flagAddedFeat = rep.batch_discover(td_errors, phiMatr, states)
    assert flagAddedFeat  # should have added at least one
    assert rep.selected_features[-1] == 9  # feat conj that yields state [0,2]
    assert rep.selected_features[-2] == 11  # feat conj that yields state [0,0]

    # Ensure that discovered features are now active
    true_phi_s1 = np.zeros(rep.features_num)
    true_phi_s1[0] = True
    true_phi_s1[4] = True  # TODO - could be [4] depending on axes, check.
    true_phi_s1[10] = True  # The conjunction of [0,0]
    assert np.all(true_phi_s1 == rep.phi_non_terminal(states[0, :]))

    true_phi_s2 = np.zeros(rep.features_num)
    true_phi_s2[0] = True
    true_phi_s2[6] = True  # TODO - could be [4] depending on axes, check.
    # The conjunction of [0,2] [[note actual id is 11, but in index 10]]
    true_phi_s2[9] = True
    assert np.all(true_phi_s2 == rep.phi_non_terminal(states[1, :]))

ext: py

max_stars_repo_name: kngwyu/rlpy

max_issues_repo_name: kngwyu/rlpy

reason: import numpy


text:
"""
The classes in this file are example time profiles that can be used in the
analysis classes. There is also GenericProfile, an abstract parent class to
inherit from to create other time profiles.
"""

__author__ = 'John Evans'
__copyright__ = 'Copyright 2020 John Evans'
__credits__ = ['John Evans', 'Jason Fan', 'Michael Larson']
__license__ = 'Apache License 2.0'
__version__ = '0.0.1'
__maintainer__ = 'John Evans'
__email__ = 'john.evans@icecube.wisc.edu'
__status__ = 'Development'

from typing import Optional, Union

import abc
import numpy as np

class BaseSpectrum:
    """A generic base class to standardize the methods for the Spectrum.

    Any callable function will work.

    Attributes:
    """

    __metaclass__ = abc.ABCMeta

    @abc.abstractmethod
    def __init__(self) -> None:
        """Initializes the Spectrum.
        """

    @abc.abstractmethod
    def __call__(self, energy: Union[np.ndarray, float],
                 **kwargs) -> np.ndarray:
        """return the differential flux at given energy(s).

        Args:
            energy: An array of energy

        Returns:
            A numpy array of differential flux.
        """

    @abc.abstractmethod
    def __str__(self) -> None:
        """String representation
        """
        return 'Base spectrum'

class PowerLaw(BaseSpectrum):
    """Spectrum class for PowerLaw.

    Use this to produce PowerLaw spectrum.

    Attributes:
        E0 (float): pivot energy
        A (float): Flux Norm
        gamma(float): Spectral index
        Ecut(float): Cut-off energy
    """

    def __init__(self, energy_0: float, flux_norm: float, gamma: float,
                 energy_cut: Optional[float] = None) -> None:
        """ Constructor of PowerLaw object.

        Args:
            energy_0: Normalize Energy
            flux_norm: Flux Normalization
            gamma: Spectral index
            energy_cut: Cut-off energy
        """

        super().__init__()
        self.energy_0 = energy_0
        self.flux_norm = flux_norm
        self.gamma = gamma
        self.energy_cut = energy_cut

    def __call__(self, energy: Union[np.ndarray, float],
                 **kwargs) -> np.ndarray:
        """Evaluate spectrum at energy E according to

                 dN/dE = A (E / E0)^gamma

        where A has units of events / (GeV cm^2 s). We treat
        the 'events' in the numerator as implicit and say the
        units are [GeV^-1 cm^-2 s^-1]. Specifying Ecut provides
        an optional spectral cutoff according to

                 dN/dE = A (E / E0)^gamma * exp( -E/Ecut )

        Args:
            energy : Evaluation energy [GeV]

        Returns:
            np.ndarray of differential flux
        """
        flux_norm = kwargs.pop('flux_norm', self.flux_norm)
        energy_0 = kwargs.pop('energy_0', self.energy_0)
        energy_cut = kwargs.pop('energy_cut', self.energy_cut)
        gamma = kwargs.pop('gamma', self.gamma)

        flux = flux_norm * (energy / energy_0)**(gamma)

        # apply optional exponential cutoff
        if energy_cut is not None:
            flux *= np.exp(-energy / self.energy_cut)

        return flux

    def __str__(self) -> None:
        """String representation
        """
        return 'PowerLaw'

class CustomSpectrum(BaseSpectrum):
    '''Custom spectrum using callable object
    '''
    def __init__(self, spectrum):
        """Constructor of CustomSpectrum object.

        Constructor

        Args:
            spectrum: Any callable object

        """

        super().__init__()
        self.spectrum = spectrum

    def __call__(self, energy: Union[np.ndarray, float],
                 **kwargs) -> np.ndarray:
        """Evaluate spectrum at energy E

        Constructor

        Args:
            energy : Evaluation energy

        Returns:
            np.ndarray of differential flux
        """
        return self.spectrum(energy)

    def __str__(self):
        r"""String representation of class"""
        return 'CustomSpectrum'

max_stars_repo_name: thejevans/mla

max_issues_repo_name: thejevans/mla

max_forks_repo_name: thejevans/mla


text:
import sys

from sorts import *
sys.path.append("..")

import random
import timeit
import math
import matplotlib.pyplot as plt
import numpy as np

sys.setrecursionlimit(1500)

def create_near_sorted_list(n, factor):
    L = create_random_list(n)
    L.sort()
    for _ in range(math.ceil(n*factor)):
        index1 = random.randint(0, n-1)
        index2 = random.randint(0, n-1)
        L[index1], L[index2] = L[index2], L[index1]
    return L

def create_random_list(n):
    L = []
    for _ in range(n):
        L.append(random.randint(1,n))
    return L

####################################################################################

# plot for quad_pivot_quicksort()
ns = [10*i for i in range(100)]
ts = []

runs = 10
for n in ns:
    arr = create_random_list(n)
    l = 0
    h = len(arr) - 1
    total = 0
    for _ in range(runs):
        start = timeit.default_timer()
        build_heap1(arr)
        end = timeit.default_timer()
        total += (end - start)
    ts.append(total / runs)
plt.plot(ns, ts, '.', label='mergesort() running time')
plt.xlabel("array length (n)")
plt.ylabel("time (t)")
plt.title("measure the performance of mergesort()")
plt.legend(loc = "upper left")

plt.show()

####################################################################################

ns = [10*i for i in range(100)]
ts = []

runs = 10
for n in ns:
    arr = create_random_list(n)
    l = 0
    h = len(arr) - 1
    total = 0
    for _ in range(runs):
        start = timeit.default_timer()
        mergesort_bottom(arr)
        end = timeit.default_timer()
        total += (end - start)
    ts.append(total / runs)
plt.plot(ns, ts, '.', label='mergesort_bottom() running time')
plt.xlabel("array length (n)")
plt.ylabel("time (t)")
plt.title("measure the performance of mergesort_bottom()")
plt.legend(loc = "upper left")

plt.show()

####################################################################################

ns = [10*i for i in range(100)]
ts = []

runs = 10
for n in ns:
    arr = create_random_list(n)
    for _ in range(runs):
        start = timeit.default_timer()
        mergesort_three(arr)
        end = timeit.default_timer()
        total += (end - start)
    ts.append(total / runs)
plt.plot(ns, ts, '.', label='mergesort_three() running time')
plt.xlabel("array length (n)")
plt.ylabel("time (t)")
plt.title("measure the performance of mergesort_three()")
plt.legend(loc = "upper left")

plt.show()

####################################################################################

####################################################################################
#WORST CASE EXPERIMENT
####################################################################################

####################################################################################

factors = np.linspace(0,0.5,1000)
ts = []

runs = 100
for n in factors:
    total = 0
    arr = create_near_sorted_list(100, n)
    for _ in range(runs):
        start = timeit.default_timer()
        mergesort_bottom(arr)
        end = timeit.default_timer()
        total += (end - start)
    ts.append(total / runs)
plt.plot(factors, ts, '.', label='mergesort_bottom() @ list length = 100')
plt.xlabel("factor")
plt.ylabel("time (t)")
plt.title("measure the performance of mergesort_bottom() random case")
plt.legend(loc = "upper left")

plt.show()

####################################################################################

factors = np.linspace(0,0.5,1000)
ts = []

runs = 100
for n in factors:
    total = 0
    arr = create_random_list(100)
    for _ in range(runs):
        start = timeit.default_timer()
        mergesort_bottom(arr)
        end = timeit.default_timer()
        total += (end - start)
    ts.append(total / runs)
plt.plot(factors, ts, '.', label='mergesort_bottom() @ list length = 100X')
plt.xlabel("factor")
plt.ylabel("time (t)")
plt.title("measure the performance of mergesort_bottom() worst-case")
plt.legend(loc = "upper left")

plt.show()

max_stars_repo_path: Sorting_Algorithms/Multipivot Mergesort and Testing/code.py


text:
# coding: utf-8
'''
Demo of the `streamplot` function.

A streamplot, or streamline plot, is used to display 2D vector fields.
'''

import numpy as np
import matplotlib.pyplot as plt

Y, X = np.mgrid[-3:3:100j, -3:3:100j]
U = -1 - X**2 + Y
V = 1 + X - Y**2
speed = np.sqrt(U*U + V*V)
plt.streamplot(X, Y, U, V, color=U, linewidth=2, cmap=plt.cm.autumn)
plt.colorbar()
plt.show()

max_stars_repo_name: timkphd/examples

max_issues_repo_name: timkphd/examples

max_forks_repo_name: timkphd/examples


text:
import pytz
import numpy as np
import datetime as dt
from flask import jsonify, request
from flask_login import current_user, login_required
from app.api import bp, mongodb

@bp.route('/rand/')
def get_abstract():
    message = "Welcome to the API :)"
    # doi_fields = list(mongodb.Corpus.find({'_id': {'$regex': '10-*'}},
    #                                                    {'_id':1}))
    #
    # doi_ids = []
    # for doi in doi_fields:
    #     doi_ids.append(doi['_id'])
    # doi_idx = np.random.choice(np.arange(len(doi_ids)), size=1)[0]
    # rand_entry = mongodb.Corpus.find_one({'_id': doi_ids[doi_idx]})
    # rand_entities = mongodb.WebLabeling.find_one({'DOI': doi_ids[doi_idx]})['Entities']
    rand_entry = {'Title': 'Simple method for determination of paraquat in plasma and serum of human patients by high-performance liquid chromatography',
                  'DOI': '10.1016/S1570-0232(02)00245-3',
                  'Abstract': "A simple and fast HPLC system is presented for quantifying paraquat in human plasma and serum using 1,1′-diethyl-4,4′-bipyridyldiylium (diethyl paraquat) as an internal standard. An octadecyl-silica column is used with an eluent of 10% acetonitrile (v/v) containing sodium 1-octanesulphonic acid (3.0 mM) and a diethylamine-orthophosphoric acid buffer (pH 3). Unlike with other techniques, sample treatment requires only the precipitation of protein contents by 6% perchloric acid (v/v) in methanol. The method has a limit of detection of 0.1 μg/ml and is linear up to 10 μg/ml. The serum of four patients and the plasma of one patient with paraquat intoxication's were analysed and positive identification and quantification was readily achieved. One of those patients survived, partially given the rapid disclosure of his levels of paraquat. Therefore, this method is suitable for quantification of paraquat in toxicological samples. It may be used as a prognostic tool in critical case detoxification and to quickly identify potentially salvageable patients for enrolment in new hemofiltration studies."}
    rand_entities = [['entity mol', 59, 67], ['entity mol', 100, 134], ['entity mol', 136, 152], ['entity cpt', 160, 177], ['entity mol', 182, 198], ['entity mol', 236, 248], ['entity mol', 266, 295], ['entity mol', 311, 344], ['entity mol', 465, 480], ['entity mol', 490, 498], ['entity cpt', 517, 535], ['entity mol', 641, 649], ['entity mol', 834, 842], ['entity mol', 901, 909]]
    content = {
        "title": rand_entry['Title'],
        "doi": rand_entry['DOI'],
        "text": rand_entry['Abstract'],
        "entities": rand_entities
    }
    status_dict = {
        "status": 200,
        "success": True,
        "message": message,
        "contentType": 'application/json',
        "content": content
    }
    return jsonify(status_dict), status_dict['status']

@bp.route('/doi_suggestions/')
def get_doi_suggestions():
    dois = []
    with open('flame_retardant_dois.txt', 'r') as f:
        for line in f:
            dois.append(line)
    message = "Welcome to the API :)"
    content = {
        "dois": dois
    }
    status_dict = {
        "status": 200,
        "success": True,
        "message": message,
        "contentType": 'application/json',
        "content": content
    }
    return jsonify(status_dict), status_dict['status']

@bp.route('/<doi>/')
def get_db_entry(doi):
    doi = doi.replace('*', '/')
    message = "You wanna DOI? Here's ya stinking DOI"
    entry = mongodb.Corpus.find_one({'_id': doi})
    title = entry['Title']
    abstract = entry['Abstract']
    try:
        fulltext = entry['FullText']
    except KeyError:
        fulltext = {'Body': [], 'RefCites': []}
    try:
        keywords = entry['Keywords']
    except KeyError:
        keywords = []
    metadata = entry['MetaData']
    content = {
        "title": title,
        "doi": doi,
        "abstract": abstract,
        "fulltext": fulltext,
        "keywords": keywords,
        "metadata": metadata
    }
    status_dict = {
        "status": 200,
        "success": True,
        "message": message,
        "contentType": 'application/json',
        "content": content
    }
    return jsonify(status_dict), status_dict['status']

@bp.route('/post_classify/', methods=['POST'])
@login_required
def get_classify():
    classify_data = request.get_json()
    doi = classify_data['doi']
    cat = classify_data['catSelect'].lower()
    rel = classify_data['relSelect']
    if rel == 'zero':
        rel = 0
    elif rel == 'one':
        rel = 1
    elif rel == 'two':
        rel = 2
    timestamp = classify_data['timeStamp']
    utc_datetime = dt.datetime.utcfromtimestamp(timestamp / 1000.)
    timestamp = utc_datetime.replace(tzinfo=pytz.timezone('UTC')).astimezone(pytz.timezone('US/Pacific'))
    classifications = mongodb.Classifications
    entry = {'DOI': doi,
             'Category': cat,
             'Relevance': rel,
             'User': current_user.username,
             'Timestamp': timestamp}
    classifications.insert_one(entry)
    return "Success", 201

max_stars_repo_name: oriondollar/beto-api

max_issues_repo_name: oriondollar/beto-api

max_forks_repo_name: oriondollar/beto-api


text:
# coding=utf-8
# Copyright 2020 The Mesh TensorFlow Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tests for Mesh TensorFlow layers."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from absl.testing import parameterized

import mesh_tensorflow as mtf
import numpy as np

import tensorflow.compat.v1 as tf
from tensorflow.python.framework import test_util  # pylint:disable=g-direct-tensorflow-import

class LayersTest(parameterized.TestCase, tf.test.TestCase):

  @parameterized.parameters(
      (4, True, "not_channels"),
      (8, False, "channels"),
  )
  def testDense(self, units, use_bias, new_dim_name):
    batch = 2
    channels = 3
    inputs = tf.random_normal([batch, channels])

    graph = mtf.Graph()
    mesh = mtf.Mesh(graph, "my_mesh")
    batch_dim = mtf.Dimension("batch", batch)
    channels_dim = mtf.Dimension("channels", channels)
    new_dim = mtf.Dimension(new_dim_name, units)

    mtf_inputs = mtf.import_tf_tensor(
        mesh, inputs, shape=mtf.Shape([batch_dim, channels_dim]))
    mtf_outputs = mtf.layers.dense(
        mtf_inputs,
        new_dims=new_dim,
        reduced_dims=[channels_dim],
        activation=mtf.relu,
        use_bias=use_bias)
    mesh_impl = mtf.placement_mesh_impl.PlacementMeshImpl(
        shape=[], layout={}, devices=[""])
    lowering = mtf.Lowering(graph, {mesh: mesh_impl})
    actual_outputs = lowering.export_to_tf_tensor(mtf_outputs)

    expected_outputs = tf.keras.layers.Dense(units=units,
                                             activation=tf.nn.relu,
                                             use_bias=use_bias)(inputs)
    tf_group = lowering.copy_masters_to_slices()
    init = tf.global_variables_initializer()
    self.evaluate(init)
    self.evaluate(tf_group)
    actual, expected = self.evaluate([actual_outputs, expected_outputs])

    self.assertEqual(actual.shape, expected.shape)

  @test_util.run_in_graph_and_eager_modes()
  def testLayerNorm(self):
    batch = 2
    channels = 3
    inputs = tf.random_normal([batch, channels])

    graph = mtf.Graph()
    mesh = mtf.Mesh(graph, "my_mesh")
    batch_dim = mtf.Dimension("batch", batch)
    channels_dim = mtf.Dimension("channels", channels)

    mtf_inputs = mtf.import_tf_tensor(
        mesh, inputs, shape=mtf.Shape([batch_dim, channels_dim]))
    mtf_outputs = mtf.layers.layer_norm(mtf_inputs,
                                        dim=channels_dim)
    mesh_impl = mtf.placement_mesh_impl.PlacementMeshImpl(
        shape=[], layout={}, devices=[""])
    lowering = mtf.Lowering(graph, {mesh: mesh_impl})
    actual_outputs = lowering.export_to_tf_tensor(mtf_outputs)

    expected_outputs = tf.keras.layers.LayerNormalization()(inputs)
    tf_group = lowering.copy_masters_to_slices()
    init = tf.global_variables_initializer()
    self.evaluate(init)
    self.evaluate(tf_group)
    actual, expected = self.evaluate([actual_outputs, expected_outputs])

    self.assertEqual(actual.shape, expected.shape)

  @test_util.run_in_graph_and_eager_modes()
  def testBatchNorm(self):
    batch = 2
    channels = 3
    inputs = tf.constant([[0, 1, 2], [4, 5, 6]], dtype=np.float32)

    graph = mtf.Graph()
    mesh = mtf.Mesh(graph, "my_mesh")
    batch_dim = mtf.Dimension("batch", batch)
    channels_dim = mtf.Dimension("channels", channels)

    mtf_inputs = mtf.import_tf_tensor(
        mesh, inputs, shape=mtf.Shape([batch_dim, channels_dim]))

    mtf_outputs_0, _ = mtf.layers.batch_norm(
        mtf_inputs,
        is_training=True, momentum=0.95, epsilon=1e-6,
        dims_idx_start=0, dims_idx_end=1, name="bn0")
    mtf_outputs_1, _ = mtf.layers.batch_norm(
        mtf_outputs_0 * 2 + 1,
        is_training=True, momentum=0.95, epsilon=1e-6,
        dims_idx_start=0, dims_idx_end=1, name="bn1")

    mesh_impl = mtf.placement_mesh_impl.PlacementMeshImpl(
        shape=[], layout={}, devices=[""])
    lowering = mtf.Lowering(graph, {mesh: mesh_impl})

    actual_outputs_0 = lowering.export_to_tf_tensor(mtf_outputs_0)
    actual_outputs_1 = lowering.export_to_tf_tensor(mtf_outputs_1)

    tf_group = lowering.copy_masters_to_slices()
    init = tf.global_variables_initializer()
    self.evaluate(init)
    self.evaluate(tf_group)
    [actual_0, actual_1] = self.evaluate([actual_outputs_0, actual_outputs_1])

    expected = np.array([[-1, -1, -1], [1, 1, 1]])
    self.assertAllClose(actual_0, expected)
    self.assertAllClose(actual_1, expected)

  @test_util.run_in_graph_and_eager_modes()
  def testWeightsNonzero(self):
    inputs = tf.constant([[3, 1, 0], [1, 0, 0]])

    graph = mtf.Graph()
    mesh = mtf.Mesh(graph, "my_mesh")
    batch_dim = mtf.Dimension("batch", inputs.shape.as_list()[0])
    channels_dim = mtf.Dimension("channels", inputs.shape.as_list()[1])

    mtf_inputs = mtf.import_tf_tensor(
        mesh, inputs, shape=mtf.Shape([batch_dim, channels_dim]))
    mtf_outputs = mtf.layers.weights_nonzero(mtf_inputs)
    mesh_impl = mtf.placement_mesh_impl.PlacementMeshImpl(
        shape=[], layout={}, devices=[""])
    lowering = mtf.Lowering(graph, {mesh: mesh_impl})
    actual_outputs = lowering.export_to_tf_tensor(mtf_outputs)

    expected_outputs = tf.cast(tf.not_equal(inputs, 0), tf.float32)
    tf_group = lowering.copy_masters_to_slices()
    self.evaluate(tf_group)
    actual, expected = self.evaluate([actual_outputs, expected_outputs])

    self.assertAllEqual(actual, expected)

  @test_util.run_in_graph_and_eager_modes()
  def testDenseReluDense(self):
    batch = 2
    channels = 3
    hidden = 5
    inputs = tf.random_normal([batch, channels])

    graph = mtf.Graph()
    mesh = mtf.Mesh(graph, "my_mesh")
    batch_dim = mtf.Dimension("batch", batch)
    channels_dim = mtf.Dimension("channels", channels)
    hidden_dim = mtf.Dimension("hidden", hidden)

    mtf_inputs = mtf.import_tf_tensor(
        mesh, inputs, shape=mtf.Shape([batch_dim, channels_dim]))
    mtf_outputs = mtf.layers.dense_relu_dense(mtf_inputs,
                                              hidden_channels=hidden_dim)
    mesh_impl = mtf.placement_mesh_impl.PlacementMeshImpl(
        shape=[], layout={}, devices=[""])
    lowering = mtf.Lowering(graph, {mesh: mesh_impl})
    actual_outputs = lowering.export_to_tf_tensor(mtf_outputs)

    tf_group = lowering.copy_masters_to_slices()
    init = tf.global_variables_initializer()
    self.evaluate(init)
    self.evaluate(tf_group)
    actual = self.evaluate(actual_outputs)

    self.assertEqual(actual.shape, inputs.shape)

  @parameterized.parameters(
      (2, 16, 3, 4, 2, 2),
      (1, 8, 5, 3, 1, 4),
  )
  def testMaskedLocalAttention1D(self, batch, length, io_channels, kv_channels,
                                 heads, window_size):
    length_q = length
    query = tf.random_normal([batch, length_q, io_channels])

    graph = mtf.Graph()
    mesh = mtf.Mesh(graph, "my_mesh")
    batch_dim = mtf.Dimension("batch", batch)
    length_q_dim = mtf.Dimension("length_q", length_q)
    io_channels_dim = mtf.Dimension("io_channels", io_channels)
    kv_channels_dim = mtf.Dimension("kv_channels", kv_channels)
    heads_dim = mtf.Dimension("heads", heads)

    mtf_query = mtf.import_tf_tensor(
        mesh, query,
        shape=mtf.Shape([batch_dim, length_q_dim, io_channels_dim]))
    mtf_outputs = mtf.layers.masked_local_attention_1d(
        mtf_query,
        kv_channels=kv_channels_dim,
        heads=heads_dim,
        window_size=window_size)
    mesh_impl = mtf.placement_mesh_impl.PlacementMeshImpl(
        shape=[], layout={}, devices=[""])
    lowering = mtf.Lowering(graph, {mesh: mesh_impl})
    actual_outputs = lowering.export_to_tf_tensor(mtf_outputs)

    tf_group = lowering.copy_masters_to_slices()
    init = tf.global_variables_initializer()
    self.evaluate(init)
    self.evaluate(tf_group)
    actual = self.evaluate(actual_outputs)

    self.assertEqual(actual.shape, (batch, length_q, io_channels))

  @parameterized.parameters(
      (2, 4, 5, 7, 3, 1),
  )
  def testDotProductAttention(
      self, batch, heads, length_q, length_kv, depth_k, depth_v):
    query = tf.random_normal([batch, heads, length_q, depth_k])
    key = tf.random_normal([batch, heads, length_kv, depth_k])
    value = tf.random_normal([batch, heads, length_kv, depth_v])

    graph = mtf.Graph()
    mesh = mtf.Mesh(graph, "my_mesh")
    batch_dim = mtf.Dimension("batch", batch)
    heads_dim = mtf.Dimension("heads", heads)
    length_q_dim = mtf.Dimension("length_q", length_q)
    length_kv_dim = mtf.Dimension("length_kv", length_kv)
    depth_k_dim = mtf.Dimension("depth_k", depth_k)
    depth_v_dim = mtf.Dimension("depth_v", depth_v)

    mtf_query = mtf.import_tf_tensor(
        mesh, query,
        shape=mtf.Shape(
            [batch_dim, heads_dim, length_q_dim, depth_k_dim]))
    mtf_key = mtf.import_tf_tensor(
        mesh, key,
        shape=mtf.Shape(
            [batch_dim, heads_dim, length_kv_dim, depth_k_dim]))
    mtf_value = mtf.import_tf_tensor(
        mesh, value,
        shape=mtf.Shape(
            [batch_dim, heads_dim, length_kv_dim, depth_v_dim]))
    mtf_outputs = mtf.layers.dot_product_attention(
        mtf_query,
        mtf_key,
        mtf_value,
        mask=None)
    mesh_impl = mtf.placement_mesh_impl.PlacementMeshImpl(
        shape=[], layout={}, devices=[""])
    lowering = mtf.Lowering(graph, {mesh: mesh_impl})
    actual_outputs = lowering.export_to_tf_tensor(mtf_outputs)

    tf_group = lowering.copy_masters_to_slices()
    init = tf.global_variables_initializer()
    self.evaluate(init)
    self.evaluate(tf_group)
    actual = self.evaluate(actual_outputs)

    self.assertEqual(actual.shape, (batch, heads, length_q, depth_v))

  @parameterized.parameters(
      (16, 4),
      (32, 8),
  )
  def testMultiheadAttention(self, kv_channels, heads):
    batch = 2
    length = 8
    channels = 3
    query = tf.random_normal([batch, length, channels])

    graph = mtf.Graph()
    mesh = mtf.Mesh(graph, "my_mesh")
    batch_dim = mtf.Dimension("batch", batch)
    length_dim = mtf.Dimension("length", length)
    channels_dim = mtf.Dimension("channels", channels)
    kv_channels_dim = mtf.Dimension("kv_channels", kv_channels)
    heads_dim = mtf.Dimension("heads", heads)

    mtf_query = mtf.import_tf_tensor(
        mesh, query,
        shape=mtf.Shape([batch_dim, length_dim, channels_dim]))
    mtf_outputs = mtf.layers.multihead_attention(
        mtf_query,
        memory_antecedent=None,
        mask=None,
        kv_channels=kv_channels_dim,
        heads=heads_dim)
    mesh_impl = mtf.placement_mesh_impl.PlacementMeshImpl(
        shape=[], layout={}, devices=[""])
    lowering = mtf.Lowering(graph, {mesh: mesh_impl})
    actual_outputs = lowering.export_to_tf_tensor(mtf_outputs)

    tf_group = lowering.copy_masters_to_slices()
    init = tf.global_variables_initializer()
    self.evaluate(init)
    self.evaluate(tf_group)
    actual = self.evaluate(actual_outputs)

    self.assertEqual(actual.shape, query.shape)

  @parameterized.parameters(
      ("MAX_2D",), ("AVG_2D",), ("MAX_3D",), ("AVG_3D",),
  )
  def testPool(self, pooling_method):
    batch = 2
    depth = 3
    height = 4
    width = 6
    channels = 3
    tf.random.set_random_seed(1234)
    inputs = tf.random_normal([batch, depth, height, width, channels])

    stride_d = 3
    stride_h = 2
    stride_w = 3

    graph = mtf.Graph()
    mesh = mtf.Mesh(graph, "my_mesh")
    batch_dim = mtf.Dimension("batch", batch)
    depth_dim = mtf.Dimension("depth", depth)
    height_dim = mtf.Dimension("height", height)
    width_dim = mtf.Dimension("width", width)
    channels_dim = mtf.Dimension("channels", channels)

    mtf_inputs = mtf.import_tf_tensor(
        mesh, inputs, shape=mtf.Shape(
            [batch_dim, depth_dim, height_dim, width_dim, channels_dim]))

    if pooling_method == "MAX_2D":
      mtf_outputs = mtf.layers.max_pool2d(
          mtf_inputs, ksize=(stride_h, stride_w))
      inputs = tf.reshape(inputs, [batch * depth, height, width, channels])
      expected_outputs = tf.keras.layers.MaxPooling2D(
          (stride_h, stride_w))(inputs)
      expected_outputs = tf.reshape(
          expected_outputs,
          [batch, depth, int(height / stride_h),
           int(width / stride_w), channels])

    elif pooling_method == "AVG_2D":
      mtf_outputs = mtf.layers.avg_pool2d(
          mtf_inputs, ksize=(stride_h, stride_w))
      inputs = tf.reshape(inputs, [batch * depth, height, width, channels])
      expected_outputs = tf.keras.layers.AveragePooling2D(
          (stride_h, stride_w))(inputs)
      expected_outputs = tf.reshape(
          expected_outputs,
          [batch, depth, int(height / stride_h),
           int(width / stride_w), channels])

    elif pooling_method == "MAX_3D":
      mtf_outputs = mtf.layers.max_pool3d(
          mtf_inputs, ksize=[stride_d, stride_h, stride_w])
      expected_outputs = tf.keras.layers.MaxPooling3D(
          [stride_d, stride_h, stride_w])(inputs)

    elif pooling_method == "AVG_3D":
      mtf_outputs = mtf.layers.avg_pool3d(
          mtf_inputs, ksize=[stride_d, stride_h, stride_w])
      expected_outputs = tf.keras.layers.AveragePooling3D(
          [stride_d, stride_h, stride_w])(inputs)

    mtf_gradient = mtf.gradients([mtf_outputs], [mtf_inputs])[0]

    mesh_impl = mtf.placement_mesh_impl.PlacementMeshImpl(
        shape=[], layout={}, devices=[""])
    lowering = mtf.Lowering(graph, {mesh: mesh_impl})
    actual_outputs = lowering.export_to_tf_tensor(mtf_outputs)
    actual_gradient = lowering.export_to_tf_tensor(mtf_gradient)

    tf_group = lowering.copy_masters_to_slices()
    init = tf.global_variables_initializer()
    self.evaluate(init)
    self.evaluate(tf_group)
    actual, expected = self.evaluate([actual_outputs, expected_outputs])
    self.assertAllClose(actual, expected)

    actual = self.evaluate(actual_gradient)
    if pooling_method == "MAX_2D":
      expected_non_zeros = batch * depth * height * width * channels / (
          stride_h * stride_w)
      self.assertEqual(np.count_nonzero(actual), expected_non_zeros)

    elif pooling_method == "AVG_2D":
      expected = np.ones((batch, depth, height, width, channels),
                         dtype=np.float32) / stride_h / stride_w
      self.assertAllClose(actual, expected)

    elif pooling_method == "MAX_3D":
      expected_non_zeros = batch * depth * height * width * channels / (
          stride_d * stride_h * stride_w)
      self.assertEqual(np.count_nonzero(actual), expected_non_zeros)

    elif pooling_method == "AVG_3D":
      expected = np.ones((batch, depth, height, width, channels),
                         dtype=np.float32) / stride_d / stride_h / stride_w
      self.assertAllClose(actual, expected)

if __name__ == "__main__":
  tf.disable_v2_behavior()
  tf.enable_eager_execution()
  tf.test.main()


text:
"""

@author Jianxing Feng
"""
import numpy as np
import theano
import theano.tensor as T
import logging
import time
import os
import datetime
#import cPickle as pickle
import pickle
from collections import OrderedDict
from theano.ifelse import ifelse
import sys
import argparse
import random
import TFImputeModel
import util

logging.basicConfig(level=logging.DEBUG,
                    format='%(asctime)s %(levelname)s %(message)s',
                    filemode='w')
logger = logging

mode = theano.Mode(linker='cvm')

model_config = 0

class Model:
    def __init__(self):
        global model_config
        cleaned_embed = [each for each in model_config.embed if each[0] != 0]
        max_motif_len = int(model_config.max_motif_len)
        seq_len = model_config.seq_len + 2 * (max_motif_len - 1)

        # Assume the 3rd column is for cell type
        tfcell_comb = []
        if (model_config.bp_comb):
            tfcell_comb = [each.strip().split() for each in open(model_config.bp_comb)]
        self.tfcell_comb = tfcell_comb

        if (model_config.cnn == "TFImputeModel"):
            self.cnn = TFImputeModel.TFImputeModel(num_motif = model_config.num_motif,
                                       max_motif_len = int(model_config.max_motif_len),
                                       embed = cleaned_embed,
                                       L1_reg = model_config.L1_reg,
                                       L2_reg = model_config.L2_reg,
                                       gradient_clip = 0,
                                       param_clip = 0,
                                       optimizer = model_config.optimizer,
                                       seq_len = seq_len,
                                       tfcell_comb_cnt = len(self.tfcell_comb))
        elif (model_config.cnn == "TFImputeModelRNN"):
            self.cnn = TFImputeModel.TFImputeModelRNN(num_motif = model_config.num_motif,
                                       max_motif_len = int(model_config.max_motif_len),
                                       embed = cleaned_embed,
                                       L1_reg = model_config.L1_reg,
                                       L2_reg = model_config.L2_reg,
                                       gradient_clip = 0,
                                       param_clip = 0,
                                       optimizer = model_config.optimizer,
                                       batch_size = model_config.batch_size * 2)
        elif (model_config.cnn == "TFImputeModelRNNMask"):
            self.cnn = TFImputeModel.TFImputeModelRNNMask(num_motif = model_config.num_motif,
                                       max_motif_len = int(model_config.max_motif_len),
                                       embed = cleaned_embed,
                                       L1_reg = model_config.L1_reg,
                                       L2_reg = model_config.L2_reg,
                                       gradient_clip = 0,
                                       param_clip = 0,
                                       optimizer = model_config.optimizer,
                                       batch_size = model_config.batch_size * 2)

        else:
            sys.stderr.write("Wrong model " + model_config.cnn + "\n")
            exit(1)

        self.token2idx = [{} for each in model_config.embed]

    def get_state(self):
        return [self.token2idx, self.cnn.get_state()]

    def set_state(self, state):
        self.token2idx = state[0]
        self.cnn.set_state(state[1])

    def seq2nparray(self, dnasequence, seq_len):
        if (model_config.train != None and model_config.min_len > 0):
            rand_len = random.randint(model_config.min_len, seq_len)
            border = (len(dnasequence) - rand_len) // 2
            if (border > 0):
                dnasequence = dnasequence[border: border + rand_len]
        if (len(dnasequence) > seq_len):
            border = (len(dnasequence) - seq_len) // 2
            dnasequence = dnasequence[border: border + seq_len]
        elif (len(dnasequence) < seq_len):
            border1 = (seq_len - len(dnasequence)) // 2
            border2 = seq_len - len(dnasequence) - border1
            dnasequence = 'N' * border1 + dnasequence + 'N' * border2

        ## pad with (max_motif_len-1) N to the border
        max_motif_len = int(model_config.max_motif_len)
        dnasequence = 'N' * (max_motif_len-1) + dnasequence + 'N' *  (max_motif_len-1)

        arr = [[0] * 4 for each in dnasequence]
        for i in range(len(dnasequence)):
            curr = dnasequence[i]
            idx = 0
            if (curr == 'A'): idx = 0
            elif (curr == 'T'): idx = 1
            elif (curr == 'C'): idx = 2
            elif (curr == 'G'): idx = 3
            elif (curr != 'N'):
                sys.stderr.write("Input error " + curr + " is encountered\n")
                exit(1)
            if (curr == 'N'): arr[i] = [0.25] * 4
            else: arr[i][idx] = 1

        return np.array(arr, dtype=theano.config.floatX)

    def complement(self, dnasequence):
        comp = ""
        for each in dnasequence:
            if (each == 'A'): comp += "T"
            if (each == 'T'): comp += "A"
            if (each == 'C'): comp += "G"
            if (each == 'G'): comp += "C"
            if (each == 'N'): comp += "N"
        return comp

    def removeN(self, seq):
        if ('N' not in seq):
            return seq
        base = 'ATCG'
        newseq = ""
        for b in seq:
            if (b != 'N'): newseq += b
            else: newseq += random.choice(base)
        return newseq

    def shuffleFile(self, datafile, buffsize = 100000):
        buf = []
        for each in open(datafile):
            if (len(buf) == buffsize):
                idx = random.randint(0, buffsize-1)
                yield buf[idx]
                buf[idx] = each.strip()
            else:
                buf.append(each.strip())
        random.shuffle(buf)
        for each in buf:
            yield each

    def scanDataBatch(self, datafile, batch_size, seq_len, shuffle_data = False):
        if (model_config.cnn == "TFImputeModelRNNMask"):
            for each in self.scanDataBatchNoShuffleMask(datafile, batch_size, seq_len):
                yield each
        else:
            for each in self.scanDataBatchNoShuffle(datafile, batch_size, seq_len):
                yield each

    def scanDataBatchNoShuffle(self, datafile, batch_size, seq_len):
        batch_size *= 2

        buffer = []
        buffer_target = []
        openf = open(datafile)

        eof = False
        while True:
            try:
                #line = openf.next()
                line = next(openf)
            except StopIteration:
                eof = True

            if (len(buffer) < batch_size and not eof):
                fields = line.strip().split("\t")
                if (len(fields) != len(model_config.embed) + 2):
                    logger.error("Format error for line %s" % line.strip())
                    continue

                idxes = []
                for i in range(len(model_config.embed)):
                    if (model_config.embed[i][0] == 0): continue
                    if (fields[i] not in self.token2idx[i]):
                        self.token2idx[i][fields[i]] = len(self.token2idx[i])
                    idxes.append(self.token2idx[i][fields[i]])

                weight = float(fields[len(model_config.embed)])
                seq = fields[len(model_config.embed)+1].upper()
                buffer.append([idxes, self.seq2nparray(seq, seq_len)])
                buffer.append([idxes, self.seq2nparray(self.complement(seq[::-1]), seq_len)])
                buffer_target.append(weight)

            if (len(buffer) == batch_size or (eof and len(buffer) > 0)):
                idxes = np.array([each[0] for each in buffer], dtype='int32')
                seq = np.dstack([each[1] for each in buffer])
                seq = np.transpose(seq, (2,1,0))
                seq = np.reshape(seq, seq.shape + (1,))
                target = np.array(buffer_target, dtype=theano.config.floatX)
                weight = np.array([1] * (len(buffer) // 2), dtype=theano.config.floatX)
                buffer = []
                buffer_target = []
                yield idxes, seq, target, weight

            if (eof): break

    def scanDataBatchNoShuffleMask(self, datafile, batch_size, seq_len):
        """
        The input format is: <CellLine><Seq><Target><Mask>
        """
        batch_size *= 2

        cleaned_embed = [each for each in model_config.embed if each[0] != 0]
        target_size = cleaned_embed[1][0]

        buffer = []
        buffer_target = []
        buffer_mask = []
        openf = open(datafile)

        eof = False
        while True:
            try:
                line = openf.next()
            except StopIteration:
                eof = True

            if (len(buffer) < batch_size and not eof):
                fields = line.strip().split("\t")
                if (len(fields) != len(model_config.embed) + 2):
                    logger.error("Format error for line %s" % line.strip())
                    continue

                # Always take only the first embed as cell line
                # The second embed is the target (TFs)
                idxes = []
                for i in range(len(model_config.embed)):
                    if (fields[i] not in self.token2idx[i]):
                        self.token2idx[i][fields[i]] = len(self.token2idx[i])
                    idxes.append(self.token2idx[i][fields[i]])
                    break

                # In case the number of target is not equal to the setted output size
                # This happens during test
                if (len(fields[2]) < target_size):
                    fields[2] = fields[2] + '0' * (target_size - len(fields[2]))
                if (len(fields[3]) < target_size):
                    fields[3] = fields[3] + '0' * (target_size - len(fields[3]))

                target = np.array([int(each) for each in fields[2]])
                mask = np.array([int(each) for each in fields[3]])
                seq = fields[1].upper()
                buffer.append([idxes, self.seq2nparray(seq, seq_len)])
                buffer.append([idxes, self.seq2nparray(self.complement(seq[::-1]), seq_len)])
                buffer_target.append(target)
                buffer_mask.append(mask)

            if (len(buffer) == batch_size or (eof and len(buffer) > 0)):
                idxes = np.array([each[0] for each in buffer], dtype='int32')
                seq = np.dstack([each[1] for each in buffer])
                seq = np.transpose(seq, (2,1,0))
                seq = np.reshape(seq, seq.shape + (1,))
                target = np.array(buffer_target, dtype='int32')
                mask = np.array(buffer_mask, dtype='int32')
                buffer = []
                buffer_target = []
                buffer_mask = []
                yield idxes, seq, target, mask

            if (eof): break

    def fit(self, trainfile, validfile, validfile2, nepochs, batch_size, seq_len):
        logger.info('... training')
        model_save_frequency = model_config.model_save_frequency / batch_size

        epoch = 0
        sum_loss = 0;
        total_cnt = 1;
        module_size = 1;
        loss_monitor_window = util.ProgressMonitor(1000, None, 'loss_monitor_window')
        last_loss_window = 0
        while (epoch < nepochs):
            epoch = epoch + 1
            for idxes, seq, target, weightOrmask in self.scanDataBatch(trainfile, batch_size, seq_len, True):
                train_loss, predict = self.cnn.train(idxes, seq, target, weightOrmask, model_config.learning_rate, model_config.momentum)
                train_loss = np.mean(train_loss)
                sum_loss += train_loss
                loss_window = loss_monitor_window.progress(train_loss)
                if (loss_window):
                    last_loss_window = loss_window[2]

                print('\rXXXXXXXXXXXXXXXXXXXXXXX INFO epoch: %s mini-batch: %s avg loss: %f, last1000 lost: %f, curr loss: %f, lr: %.6f ' % (
                    str(epoch).ljust(5), str(total_cnt).ljust(15), sum_loss
                    / total_cnt, last_loss_window, train_loss,
                    model_config.learning_rate),)
                sys.stdout.flush()
                if (total_cnt % module_size == 0):
                    print("\r",)
                    sys.stdout.flush()
                    module_size *= 2
                    logger.info('epoch: %s mini-batch: %s avg loss: %f, last1000 lost: %f, curr loss: %f, lr: %.6f ' %
                                (str(epoch).ljust(5), str(total_cnt).ljust(15), sum_loss / total_cnt, last_loss_window, train_loss, model_config.learning_rate))
                if (total_cnt % model_save_frequency == 0):
                    model_config.learning_rate *= 1-model_config.learning_rate_decay
                    if (model_config.model != None):
                        print("\r",)
                        sys.stdout.flush()
                        mf = model_config.model + "." + str(total_cnt / model_save_frequency)
                        logger.info("Save model to %s ..." % mf)
                        util.save([model_config, self.get_state()], mf)

                    if validfile != None:
                        loss = [self.cnn.error(idxes, seq, target, weightOrmask) for idxes, seq, target, weightOrmask in self.scanDataBatch(validfile, batch_size, seq_len)]
                        valid_loss = np.mean([np.mean(each[0]) for each in loss])
                        sys.stdout.flush()
                        logger.info('epoch: %i, valid loss: %f, lr: %.6f %s' % (epoch, valid_loss, model_config.learning_rate, ' ' * 30))

                    if validfile2 != None:
                        loss = [self.cnn.error(idxes, seq, target, weightOrmask) for idxes, seq, target, weightOrmask in self.scanDataBatch(validfile2, batch_size, seq_len)]
                        valid_loss = np.mean([np.mean(each[0]) for each in loss])
                        sys.stdout.flush()
                        logger.info('epoch: %i, valid2 loss: %f, lr: %.6f %s' % (epoch, valid_loss, model_config.learning_rate, ' ' * 30))

                total_cnt += 1

    def test(self, testfile, prediction, batch_size, seq_len):
        logger.info('Testing ...')
        if (prediction):
            predout = open(prediction, 'w')

        sum_loss = 0
        module_size = 1
        total_cnt = 0

        for idxes, seq, target, weightOrmask in self.scanDataBatch(testfile, batch_size, seq_len):
            total_cnt += 1
            loss, predict = self.cnn.error(idxes, seq, target, weightOrmask)
            sum_loss += np.mean(loss)
            if (total_cnt % module_size == 0):
                module_size *= 2
                logger.info('mini-batch: %s avg loss: %f' % (str(total_cnt).ljust(15), sum_loss / total_cnt))

            if (prediction):
                if (model_config.cnn == "TFImputeModelRNNMask"):
                    for each in zip(target, predict):
                        # In this case, each element of target and predict is a vector
                        # The length of both vectors could be different
                        out1 = "".join([str(e)     for e in each[0]])
                        out2 = " ".join(["%.5f" % e for e in each[1]])
                        predout.write("%s\t%s\n" % (out1, out2))
                else:
                    for each in zip(target, predict):
                        predout.write("%d\t%.5f\n" % each)

        if (prediction):
            predout.close()

    def batch_predict(self, testfile, prediction, batch_size, seq_len):
        logger.info('Batch Predicting ...')
        if (prediction):
            predout = open(prediction, 'w')

        module_size = 1
        total_cnt = 0

        self.tfcell_comb = [(self.token2idx[0][each[0]], self.token2idx[2][each[1]]) for each in self.tfcell_comb]
        self.cnn.prepare_batch_predict(self.tfcell_comb)

        for idxes, seq, target, weight in self.scanDataBatch(testfile, batch_size, seq_len):
            total_cnt += 1
            predict = self.cnn.batch_predict(seq)
            if (total_cnt % module_size == 0):
                module_size *= 2
                logger.info('mini-batch: %s' % (str(total_cnt).ljust(15)))

            if (prediction):
                i = 0
                for row in predict:
                    predout.write("%.5f" % row[0])
                    for each in row[1:]:
                        predout.write("\t%.5f" % each)
                    predout.write("\n")
                    i += 1

        if (prediction):
            predout.close()

    def get_seq_feature(self, testfile, prediction, batch_size, seq_len):
        logger.info('Get Sequence Feature ...')
        if (prediction):
            predout = open(prediction, 'w')

        module_size = 1
        total_cnt = 0

        self.cnn.prepare_batch_predict()

        for idxes, seq, target, weight in self.scanDataBatch(testfile, batch_size, seq_len):
            total_cnt += 1
            predict = self.cnn.get_seq_feature(seq)
            if (total_cnt % module_size == 0):
                module_size *= 2
                logger.info('mini-batch: %s' % (str(total_cnt).ljust(15)))

            if (prediction):
                i = 0
                for row in predict:
                    first=True
                    for each in row:
                        if (first): predout.write("%.5f" % each)
                        else:       predout.write("\t%.5f" % each)
                        first = False

                    if (i % 2 == 1): predout.write("\n")
                    else:            predout.write("\t")
                    i += 1

        if (prediction):
            predout.close()

    def print_human(self):
        embedding = self.cnn.get_embedding()
        cnt = 0
        for i in range(len(model_config.embed)):
            if (model_config.embed[i][0] == 0): continue
            pairs = self.token2idx[i].items()
            pairs = sorted(pairs, key=lambda x: x[1])
            for key,idx in pairs:
                sys.stdout.write("Embed_%d\t%s" % (i, key))
                for j in range(len(embedding[cnt][idx])):
                    #print(cnt)
                    #print(idx)
                    #print('test-----')
                    #print(embedding[cnt][idx][j])
                    sys.stdout.write("\t%s" % str(embedding[cnt][idx][j]))
                sys.stdout.write("\n")
            cnt += 1

def process_command_line(argv):
    """ Processing command line """

    if argv is None:
        argv = sys.argv[1:]

    parser = argparse.ArgumentParser(description="""Version 0.1. cnn_bind: This script build a deep learning model to model the ChIPseq peak data
                                     Execute this script using: OMP_NUM_THREADS=2 THEANO_FLAGS='device=cpu,openmp=True,exception_verbosity=high' python TFImpute.py""")

    # Manual:
    parser.add_argument("-sf", dest="seq_feature", action='store_true', help="""For each in put sequence, get the sequence feature. For each sequence, the forward and backward feature
                                                                            are concatenated.""")
    parser.add_argument("-bp", dest="batch_predict", action='store_true', help="""After model is train, do batch prediction. In this model, the TF and cell types will be ignored
                                                                  in the input. All possible TF and cell types in the model will calculated. The output for each input sequence will
                                                                  be an array with #TF * #CellLine elements. Suppose the TF is the first effective embedding and cell type is the
                                                                  second effective embedding. The the output is: TF1-Cell1 TF1-Cell2 ... """)
    parser.add_argument("-bpcomb", dest="bp_comb", help="""During batch predict, if this parameter is set, only tf cell type combinations appearing in the file will be used""")
    parser.add_argument("-ph", dest="print_human", action='store_true', help="""Print the model""")
    parser.add_argument("-ct", dest="continue_train", action='store_true', help="Load the model and continue training.")
    parser.add_argument("-train", dest="train", default=None, help="""Data for training. The format is: <TF name> <Condition> <Sequence>""")
    parser.add_argument("-valid", dest="valid", default=None, help="""Data for validation. The format is the same as -train""")
    parser.add_argument("-valid2", dest="valid2", default=None, help="""Data for validation2. The format is the same as -train""")
    parser.add_argument("-test", dest="test", default=None, help="""Data for testing. For testing, -M, -b, -p, -w will overwrite the values in loaded model and -m will be set to 1.
                                                                  many other parameters for training will not be used.""")
    parser.add_argument("-M", dest="model", default=None, help="""The trained model. If -train is specified, the model will be trained from scratch and save to the specified
                                                                  path. If -train is not specified but -test is specified, the model will be loaded to run on the test data.
                                                                  In this case, all parameters specified in the command line will be ignored except -w.""")
    parser.add_argument("-F", dest="model_save_frequency", default=1000000, type=int, help="Save the model every <model_frequency> number of instances.")
    parser.add_argument("-O", dest="optimizer", default='adadelta', help="Optimization methods. Candiates are: 'sgd' and 'adadelta'.")
    parser.add_argument("-e", dest="nepochs", default=1000, type=int, help="The number of epochs to run")
    parser.add_argument("-p", dest="prediction", default=None, help="""The file for prediction output. It is effective only when -train is not set but -test is set. Note that
                                                                the odd number output (counting from 1) is the true score and the even number output is the score of shuffled background.""")
    parser.add_argument("-cnn", dest="cnn", default="Simple", help="The cnn model used. Could be 'TFImputeModel'")
    parser.add_argument("-IW", dest="ignore_weight", action='store_true', help="Whether to ignore the weight.")
    parser.add_argument("-m", dest="batch_size", default=10, type=int, help="The number of blocks for each batch")
    parser.add_argument("-seqLen", dest="seq_len", default=101, type=int, help="The length of each DNA sequence")
    parser.add_argument("-minLen", dest="min_len", default=0, type=int, help="""The min length of each DNA sequence. If it is > 0, then a random length in [min_len, seq_len] will be
                                                                    generated for each instance. The omitted sequence will be filled with N.""")
    parser.add_argument("-mml", dest="max_motif_len", default=32, help="The max length of motif. If can also be a set of numbers separated by ,")
    parser.add_argument("-nm", dest="num_motif", default=256, help="The number of motifs. It might be multiple numbers separated by , to indicate the number of motifs for each layer.")
    parser.add_argument("-embed", dest="embed", default=None, help="""A set of number pairs separated by COMMA. Two number of each pair are separated by :
                                                                    For example  10:50,5:20,300:100 means that the first three column of the data are for embedding.
                                                                    The first column contains 10 unique tokens and embed with length 50
                                                                    The first column contains 5  unique tokens and embed with length 20
                                                                    The first column contains 300 unique tokens and embed with length 100""")
    parser.add_argument("-l", dest="learning_rate", type=float, default=1, help="Learning rate")
    parser.add_argument("-d", dest="learning_rate_decay", type=float, default=0, help="Learning rate decay of every epoch")
    parser.add_argument("-mom", dest="momentum", type=float, default=0.9, help="Momentum")

    parser.add_argument("-L1", dest="L1_reg", type=float, default=0.0, help="The weight penalty for the L1 norm")
    parser.add_argument("-L2", dest="L2_reg", type=float, default=0.0, help="The weight penalty for the L2 norm")
    parser.add_argument("-c", dest="clip", default=1, type=float, help="Clip the parameter gradient to avoid gradient exploding. Set this parameter to 0 to skip clipping")
    parser.add_argument("-C", dest="param_clip", default=0, type=float, help="Clip all the parameters avoid parameter exploding. Set this parameter to 0 to skip clipping")

    return parser.parse_args(argv)

def main(argv=None):
    global model_config
    model_config = process_command_line(argv)
    if (model_config.optimizer != 'sgd' and model_config.optimizer != 'adadelta'):
        logger.error("Wrong parameters for -O.")
        exit(1)

    t0 = time.time()
    if (model_config.train != None):
        if (model_config.embed):
            em = model_config.embed.split(",")
            em = [each.split(":") for each in em]
            model_config.embed = [[int(col) for col in row] for row in em]
        else:
            model_config.embed = [[0,0]]

        model = Model()

        if (model_config.model != None and model_config.continue_train):
            state = util.load(model_config.model)
            new_model_config = state[0]
            new_model_config.batch_size = model_config.batch_size
            new_model_config.prediction = model_config.prediction
            new_model_config.model = model_config.model
            new_model_config.train = model_config.train
            new_model_config.valid = model_config.valid
            new_model_config.valid2 = model_config.valid2
            model_config = new_model_config
            model.set_state(state[1])
        util.report_parameter(model_config)

        model.fit(model_config.train, model_config.valid, model_config.valid2, model_config.nepochs, model_config.batch_size, model_config.seq_len)
        if (model_config.model != None):
            util.save([model_config, model.get_state()], model_config.model)
    elif (model_config.test != None and model_config.model != None):
        state = util.load(model_config.model)
        new_model_config = state[0]
        new_model_config.batch_size = model_config.batch_size
        new_model_config.prediction = model_config.prediction
        new_model_config.model = model_config.model
        new_model_config.test = model_config.test
        new_model_config.batch_predict = model_config.batch_predict
        new_model_config.bp_comb = model_config.bp_comb
        new_model_config.seq_feature = model_config.seq_feature
        new_model_config.min_len = 0
        model_config = new_model_config
        util.report_parameter(model_config)

        model = Model()
        model.set_state(state[1])
        if (model_config.batch_predict):
            model.batch_predict(model_config.test, model_config.prediction, model_config.batch_size, model_config.seq_len)
        elif (model_config.seq_feature):
            model.get_seq_feature(model_config.test, model_config.prediction, model_config.batch_size, model_config.seq_len)
        else:
            model.test(model_config.test, model_config.prediction, model_config.batch_size, model_config.seq_len)

    elif (model_config.print_human):
        state = util.load(model_config.model)
        new_model_config = state[0]
        new_model_config.model = model_config.model
        new_model_config.bp_comb = model_config.bp_comb
        model_config = new_model_config
        util.report_parameter(model_config)
        model = Model()
        model.set_state(state[1])
        model.print_human()
    else:
        logger.error("Either -train is needed or -test and -M is needed")

    logger.info("Elapsed time: %f" % (time.time() - t0))

if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO)
    status = main()
    sys.exit(status)

max_stars_repo_name: qinqian/TFImpute

max_issues_repo_name: qinqian/TFImpute

max_forks_repo_name: qinqian/TFImpute

reason: import numpy,import theano,from theano


text:
#!/usr/bin/env python3
"""
Generate training data.

Usage:
    gentraining.py <datadir>

"""
import os
import glob

import docopt
import dtcwt
import imageio
import numpy as np
import skimage.measure as skmeas
import skimage.transform as sktrans

import mlmask

class ReservoirSampler(object):
    """A simple reservoir sampler which will sample k items uniformly from a
    list of unknown length. Implemented to take numpy arrays and sample rows.

    """
    def __init__(self, k):
        self.samples = None # allocated on first call to sample()
        self.k = k
        self._idx = 0 # Index of next datum

    @property
    def sample_count(self):
        return self._idx

    def sample(self, data):
        data = np.atleast_2d(data)

        if self.samples is None:
            self.samples = np.zeros(
                (self.k,) + data.shape[1:], dtype=data.dtype)

        for datum in data:
            if self._idx < self.k:
                # Fill reservoir with first k samples
                self.samples[self._idx, ...] = datum
            else:
                # Randomly replace elements in reservoid
                r = np.random.randint(0, self._idx)
                if r < self.k:
                    self.samples[r, ...] = datum

            self._idx += 1

def resize(image, shape):
    """Resize an input array."""
    sc = np.abs(image).max()
    return sktrans.resize(image / sc, shape, mode='reflect') * sc

def main():
    opts = docopt.docopt(__doc__)
    datadir = opts['<datadir>']
    output_fn = os.path.join(datadir, 'training.npz')

    training_size = 10000000
    foreground_training = ReservoirSampler(training_size)
    background_training = ReservoirSampler(training_size)

    transform = dtcwt.Transform2d()

    for input_fn in glob.glob(os.path.join(datadir, 'input', '*.JPG')):
        input_base = os.path.splitext(os.path.basename(input_fn))[0]
        trimap_fn = os.path.join(datadir, 'trimap', input_base + '.png')
        if not os.path.isfile(trimap_fn):
            continue
        superpixel_fn = os.path.join(datadir, 'superpixel', input_base + '.npz')
        if not os.path.isfile(superpixel_fn):
            continue

        print('Input: ' + input_fn)
        print('Trimap: ' + trimap_fn)
        print('Superpixel: ' + superpixel_fn)

        input_im = imageio.imread(input_fn)
        trimap_im = imageio.imread(trimap_fn)

        # Only use first channel of trimap_im
        if len(trimap_im.shape) > 2:
            trimap_im = trimap_im[..., 0]

        assert len(trimap_im.shape) == 2
        assert len(input_im.shape) == 3
        assert input_im.shape[2] == 3

        sp_labels = np.load(superpixel_fn)['labels']

        # Modify trimap with superpixels
        modified_trimap = np.copy(trimap_im)
        for props in skmeas.regionprops(sp_labels, trimap_im):
            coords = props.coords
            rows, cols = coords[:, 0], coords[:, 1]
            trimap_pxs = trimap_im[rows, cols]
            n_pxs = trimap_pxs.shape[0]
            n_fg = np.count_nonzero(trimap_pxs > 224)
            n_bg = np.count_nonzero(trimap_pxs < 32)

            # Only extend non-equivocal regions with > 10% labels
            if n_bg == 0 and n_fg * 10 >= n_pxs:
                modified_trimap[rows, cols] = 255
            elif n_fg == 0 and n_bg * 10 >= n_pxs:
                modified_trimap[rows, cols] = 0

        # Write modified trimap for inspection
        modified_trimap_fn = os.path.join(datadir, 'trimap',
            input_base + '_modified.png')
        print('Writing: {}'.format(modified_trimap_fn))
        imageio.imwrite(modified_trimap_fn, modified_trimap)

        modified_trimap_vis_fn = os.path.join(datadir, 'trimap',
            input_base + '_modified_vis.jpg')
        imageio.imwrite(modified_trimap_vis_fn,
            mlmask.visualise_trimap(input_im, modified_trimap))

        trimap_vis_fn = os.path.join(datadir, 'trimap',
            input_base + '_vis.jpg')
        imageio.imwrite(trimap_vis_fn,
            mlmask.visualise_trimap(input_im, trimap_im))

        # Extract indices of +ve and -ve training samples
        modified_trimap = modified_trimap.reshape((-1,))
        foreground_mask = modified_trimap > 224
        background_mask = modified_trimap < 32

        feature_vector = mlmask.image_to_features(input_im,
            transform=transform, nlevels=6)

        # Reshape to have one feature vector per row
        feature_vector = feature_vector.reshape((-1, feature_vector.shape[-1]))

        # Update samples
        foreground_training.sample(feature_vector[foreground_mask, :])
        background_training.sample(feature_vector[background_mask, :])

        print('Foreground sample count: {}'.format(
            foreground_training.sample_count))
        print('Background sample count: {}'.format(
            background_training.sample_count))

        # Save as we go
        print('Saving to {}'.format(output_fn))
        np.savez_compressed(output_fn,
                foreground=foreground_training.samples,
                foreground_sample_size=foreground_training.sample_count,
                background=background_training.samples,
                background_sample_size=background_training.sample_count)

if __name__ == '__main__':
    main()


text:
from typing_extensions import Concatenate
import dgl
import torch
from torch._C import _llvm_enabled
import torch.nn as nn
import torch.nn.functional as F
import itertools
import numpy as np
import scipy.sparse as sp
from dgl.nn import SAGEConv

label
item user
Concat
MLP

pred
\sigma (pred - label)^2 + \sigma (pred_i - pred_j)^2 + \lambda
loss function

positive
negative

class GraphSAGE(nn.module):
    # here h_feats_0 is number_user_features, h_feats_1 is number_item_features
    def __init__(self, in_feats, h_feats_0, h_feats_1, rel_names):
        super(GraphSAGE, self).__init__()
        # self.conv1 = GraphSAGE(h_feats_0, h_feats_0, 'mean')
        # self.conv2 = GraphSAGE(h_feats_0, h_feats_0, 'mean')
        # self.conv3 = GraphSAGE(h_feats_1, h_feats_1, 'mean')
        # self.conv4 = GraphSAGE(h_feats_1, h_feats_1, 'mean')

        self.conv0 = nn.GraphConv(h_feats_0, h_feats_0)
        self.conv1 = nn.GraphConv(h_feats_1, h_feats_1)
        self.CONV0 = nn.HeteroGraphConv({'evaluate': self.conv0, 'evaluated': self.conv1}, aggregate = 'sum')
        self.CONV1 = nn.HeteroGraphConv({'evaluate': self.conv0, 'evaluated': self.conv1}, aggregate = 'sum')

        '''
        self.conv1 = nn.HeteroGraphConv({
            rel: nn.GraphConv(in_feats, h_feats_0)
            for rel in rel_names}, aggregate = 'sum'
        )

        self.CONV0 = ({self.conv1, self.conv3})
        self.CONV1 = ({self.conv2, self.conv4})
        '''

    def forward(self, g, inputs):
        with g.local_scope():
            h = self.CONV0(g, inputs)
            h = {k: F.relu(v) for k, v in h.items()}
            h = self.CONV1(g, h)
            return h

class MLPPredictor(nn.module):
    def __init__(self, h_feats):
        super().__init__()
        self.W1 = nn.Linear(h_feats, 1)

    def apply_edges(self, edges):
        h = torch.cat([edges.src['h'], edges.dst['h']], 1)
        return {'score': self.W1(h).squeeze(1)}

    def forward(self, g, h):
        with g.local_scope():
            g.ndata['h'] = h
            g.apply_edges(self.apply_edges)
            return g.data['score']

def compute_loss(pos_score, neg_score):
    scores = torch.cat([pos_score, neg_score])
    labels = torch.cat([torch.ones(pos_score.shape[0]), torch.zeros(neg_score.shape[0])])
    return F.binary_cross_entropy_with_logits(scores, labels) + F.mse_loss(scores, scores)

from sklearn.metrics import roc_auc_score
def compute_auc(pos_score, neg_score):
    scores = torch.cat([pos_score, neg_score])
    labels = torch.cat([torch.ones(pos_score.shape[0]), torch.zeros(neg_score.shape[0])])
    print(scores)
    print(scores.shape)
    print('acc = ', labels == scores / (scores.shape[0]))
    print('roc_auc_score = ', roc_auc_score(labels, scores))
    return roc_auc_score(labels, scores)

train_g = []
number_user_features = 79
number_item_features = 152 - 79
model = GraphSAGE(train_g.ndata['features'].shape[1], number_user_features, number_item_features)
h = model(train_g, number_user_features + number_item_features, number_user_features, number_item_features)

pred = MLPPredictor(number_user_features + number_item_features)

reason: import numpy,import scipy


text:
"""
Tests using an implementation of a 5-dimensional Gaussian problem and its
Nested Sampling using MultiNest via Gleipnir.

Adapted from the DNest4 python gaussian example:
https://github.com/eggplantbren/DNest4/blob/master/python/examples/gaussian/gaussian.py
"""

import pytest
import numpy as np
from numpy import exp, log, pi
from scipy.stats import uniform
from scipy.special import erf
from gleipnir.sampled_parameter import SampledParameter
from gleipnir.dnest4 import DNest4NestedSampling
import os
import glob

# Number of paramters to sample is 5
ndim = 5
# Set up the list of sampled parameters: the prior is Uniform(-5:5) --
# we are using a fixed uniform prior from scipy.stats
sampled_parameters = [SampledParameter(name=i, prior=uniform(loc=-5.0,scale=10.0)) for i in range(ndim)]
# Set the active point population size
population_size = 100
# Number of iterations -- num_steps
num_steps = 200
# Number of iterations between creation of each diffusive level. num_per_step
num_per_step = 100
# Number of diffusive levels
n_levels = 10

# Define the loglikelihood function
def loglikelihood(sampled_parameter_vector):
    const = -0.5*np.log(2*np.pi)
    return -0.5*np.sum(sampled_parameter_vector**2) + ndim * const

width = 10.0
def analytic_log_evidence(ndim, width):
      lZ = (ndim * np.log(erf(0.5*width/np.sqrt(2)))) - (ndim * np.log(width))
      return lZ

shared = {'DNS': None}

def test_initialization():
    DNS = DNest4NestedSampling(sampled_parameters=sampled_parameters,
                                   loglikelihood=loglikelihood,
                                   population_size=population_size,
                                   n_diffusive_levels=n_levels,
                                   num_steps=num_steps,
                                   num_per_step=num_per_step)
    shared['DNS'] = DNS

def test_attributes():
    DNS = shared['DNS']
    sp = DNS.sampled_parameters
    assert sp == sampled_parameters
    lnl = DNS.loglikelihood
    spv = np.array([5.,5.,5.,5.,5.])
    assert lnl(spv) == loglikelihood(spv)
    pop = DNS.population_size
    assert pop == population_size
    diff_lev = DNS.n_diffusive_levels
    assert diff_lev == 10
    dn4back = DNS.dnest4_backend
    assert dn4back == 'memory'
    nst = DNS.dnest4_kwargs['num_steps']
    assert nst == num_steps
    nps = DNS.dnest4_kwargs['num_per_step']
    assert nps == num_per_step

def test_func_run():
    DNS = shared['DNS']
    log_evidence, log_evidence_error = DNS.run(verbose=False)
    analytic = analytic_log_evidence(ndim, width)
    print(analytic, log_evidence)
    assert np.isclose(log_evidence, analytic, rtol=1.e-1)
    shared['DNS'] = DNS

def test_properties():
    DNS = shared['DNS']
    analytic = analytic_log_evidence(ndim, width)
    lnZ = DNS.log_evidence
    assert np.isclose(lnZ, analytic, rtol=1.e-1)
    lnZ_err = DNS.log_evidence_error
    Z = DNS.evidence
    Z_err = DNS.evidence_error
    H = DNS.information
    assert H is not None

def test_func_posteriors():
    DNS = shared['DNS']
    posteriors = DNS.posteriors()
    keys = list(posteriors.keys())
    assert len(keys) == len(sampled_parameters)

def test_func_akaike_ic():
    DNS = shared['DNS']
    aic = DNS.akaike_ic()

def test_func_bayesian_ic():
    DNS = shared['DNS']
    bic = DNS.bayesian_ic(n_data=5)

def test_func_deviance_ic():
    DNS = shared['DNS']
    dic = DNS.deviance_ic()

if __name__ == '__main__':
    test_initialization()
    test_attributes()
    test_func_run()
    test_properties()
    test_func_posteriors()
    test_func_akaike_ic()
    test_func_bayesian_ic()
    test_func_deviance_ic()

max_stars_repo_name: LoLab-VU/Gleipnir

max_issues_repo_name: LoLab-VU/Gleipnir

max_forks_repo_name: LoLab-VU/Gleipnir

reason: import numpy,from numpy,from scipy


text:
import numpy as np
from scipy import special
import matplotlib.pyplot as plt
import quadpy
import math

#using a basis of l spherical harmonics
def matlab_legendre(n,X):
    res = []
    for m in range(n+1):
        res.append(np.array(special.lpmv(m,n,X)))
    return np.array(res)

#using a basis of l spherical harmonics
#value of l to use
degree=7

### parameters
#Rotational constant for a hydrogen molecule in wavenumber
B=61.20979
# B=1.0
#conversion factor from wavenumber to meV for JZL
w2meV=0.1239842

### not using at the moment
#h/2pi
hbar = 1.05457173E-34
#the mass of a hydorgen atom
Hmass = 1.6737236E-27
#the reduced mass of a hydrogen molecule
mu = Hmass*Hmass/(Hmass+Hmass)
#the bond length of a hydrogen molecule
R = 0.741*10E-10
I = mu*R**2
BB = hbar**2 / (2*I) * (6.24150913*10E18)*1000;
### not using at the moment

# to caclculate the total number of basis functions
NBas = 0
for i in range(0,degree+1):
    numfun = 2*i + 1
    NBas = NBas + numfun

# to setup the initial matrix
v = np.zeros(phi.size)
H = np.zeros([NBas, NBas],dtype=complex)
J = np.array([])
for i in range(0,degree+1):
    for j in range(0,2*i+1):
        J = np.append(J,[int(i)])

# to build the Lededev Sphere using 146 points
leb = quadpy.sphere.lebedev_019()
phi = np.arctan2(leb.points[:,1],leb.points[:,0])
theta = np.arccos(leb.points[:,2])

# to construct the Spherical Harmonics basis over theta and phi
k = (degree + 1)**2
Y = np.zeros([leb.points[:,0].size, k],dtype=complex)
for j in range(0,degree+1):
    Pm = np.matrix.transpose(matlab_legendre(j,leb.points[:,2]))
    lconstant = ((2*j +1)/(4*np.pi))**(0.5)
    #calculate where to put the vector
    center = (j+1)**2 -j
    # calculate the Yj0
    Y[:,center-1] = lconstant*Pm[:,0]
    # calculate the order Ylm of the set (if any)
    for m in range(1,j+1):
        precoeff = lconstant * (math.factorial(j-m)/math.factorial(j+m))**(0.5)
        mod = m%2
        if mod == 1:
            Y[:,center + m-1] =  precoeff*Pm[:,m]*np.exp(1j*m*phi)
            Y[:,center - m-1] = -precoeff*Pm[:,m]*np.exp(-1j*m*phi)
        else:
            Y[:,center + m-1] =  precoeff*Pm[:,m]*np.exp(1j*m*phi)
            Y[:,center - m-1] =  precoeff*Pm[:,m]*np.exp(-1j*m*phi)

# to calculate the potential matrix
## we first build the diagonal terms
H_diag = np.zeros([NBas, NBas],dtype=complex)
for n in range(0,NBas):
    H_diag[n,n]=B*J[n]*(J[n]+1)+H[n,n]
rawdata=np.array([])
## the strength of rotational barrier
b=0.0
f=open('rawdata.csv','a')
# the strength of rotational barrier
for a in np.arange(0,100,1):
    # Generte rotational barrier
    # v = a*B*(np.sin(theta)**2)+(0.5*b)*(np.cos(2*phi))*(np.sin(theta)**2) # 2-D rotational barrier
    # v = a*B*(np.sin(theta)**2) # 1-D rotational barrier
    # v = a*(np.sin(theta)**2)
    H = H_diag + np.matmul(np.matrix.getH(Y),v.reshape(-1,1)*(leb.weights*4*np.pi).reshape(-1,1)*Y)
    eigenvalues, eigenvectors= np.linalg.eig(H)
    newline=np.append(np.array(a),np.sort(eigenvalues.real)*w2meV)
    np.savetxt(f, newline, fmt='%1.3f', newline=", ")
    f.write("\n")
    #plot rotational transition
    for i in range(0,9):
        plt.plot(a*w2meV,w2meV*np.sort(eigenvalues.real)[i],'bo')

f.close()
plt.show()

reason: import numpy,from scipy


text:
import ctypes
import numpy as np
from .dlpack import _c_str_dltensor, DLManagedTensor, DLTensor

ctypes.pythonapi.PyCapsule_IsValid.restype = ctypes.c_int
ctypes.pythonapi.PyCapsule_IsValid.argtypes = [ctypes.py_object, ctypes.c_char_p]

ctypes.pythonapi.PyCapsule_GetPointer.restype = ctypes.c_void_p
ctypes.pythonapi.PyCapsule_GetPointer.argtypes = [ctypes.py_object, ctypes.c_char_p]

def _array_interface_from_dl_tensor(dlt):
    """Constructs NumPy's array_interface dictionary
    from `dlpack.DLTensor` descriptor."""
    assert isinstance(dlt, DLTensor)
    shape = tuple(dlt.shape[dim] for dim in range(dlt.ndim))
    itemsize = dlt.dtype.lanes * dlt.dtype.bits // 8
    if dlt.strides:
        strides = tuple(
            dlt.strides[dim] * itemsize for dim in range(dlt.ndim)
        )
    else:
        # Array is compact, make it numpy compatible.
        strides = []
        for i, s in enumerate(shape):
            cumulative = 1
            for e in range(i + 1, dlt.ndim):
                cumulative *= shape[e]
            strides.append(cumulative * itemsize)
        strides = tuple(strides)
    typestr = "|" + str(dlt.dtype.type_code)[0] + str(itemsize)
    return dict(
        version=3,
        shape=shape,
        strides=strides,
        data=(dlt.data, True),
        offset=dlt.byte_offset,
        typestr=typestr,
    )

class _Holder:
    """A wrapper that combines a pycapsule and array_interface for consumption by  numpy.

    Parameters
    ----------
    array_interface : dict
        A description of the underlying memory.

    pycapsule : PyCapsule
        A wrapper around the dlpack tensor that will be converted to numpy.
    """

    def __init__(self, array_interface, pycapsule) -> None:
        self.__array_interface__ = array_interface
        self._pycapsule = pycapsule

def to_numpy(pycapsule) -> np.ndarray:
    """Convert a dlpack tensor into a numpy array without copying.

    Parameters
    ----------
    pycapsule : PyCapsule
        A pycapsule wrapping a dlpack tensor that will be converted.

    Returns
    -------
    np_array : np.ndarray
        A new numpy array that uses the same underlying memory as the input
        pycapsule.
    """
    assert ctypes.pythonapi.PyCapsule_IsValid(pycapsule, _c_str_dltensor)
    dl_managed_tensor = ctypes.pythonapi.PyCapsule_GetPointer(
        pycapsule, _c_str_dltensor
    )
    dl_managed_tensor_ptr = ctypes.cast(dl_managed_tensor, ctypes.POINTER(DLManagedTensor))
    dl_managed_tensor = dl_managed_tensor_ptr.contents
    holder = _Holder(_array_interface_from_dl_tensor(dl_managed_tensor.dl_tensor), pycapsule)
    return np.ctypeslib.as_array(holder)

max_stars_repo_name: oleksandr-pavlyk/dlpack

max_issues_repo_name: oleksandr-pavlyk/dlpack

max_forks_repo_name: oleksandr-pavlyk/dlpack


text:
#! /usr/bin/env python3
# Copyright (c) 2021 Grumpy Cat Software S.L.
#
# This Source Code is licensed under the MIT 2.0 license.
# the terms can be found in  LICENSE.md at the root of
# this project, or at http://mozilla.org/MPL/2.0/.

# %%
import numpy as np
import matplotlib.pyplot as plt
import shapelets.compute as sc

def readucr(filename):
    data = np.loadtxt(filename, delimiter="\t")
    y = data[:, 0].T
    x = data[:, 1:].T
    return sc.array(x), sc.array(y).astype(int)

root_url = "https://raw.githubusercontent.com/hfawaz/cd-diagram/master/FordA/"

x_train, y_train = readucr(root_url + "FordA_TRAIN.tsv")
y_train[y_train == -1] = 0

x_test, y_test = readucr(root_url + "FordA_TEST.tsv")

# %%
results = sc.clustering.kshape(x_train[:, :400], 2, max_iterations=1000)

# %%
plt.plot(results.centroids)
plt.show()

# %%
results.labels

# %%
y_train[:100]

# %%
sc.statistics.mean(sc.square(results.labels - y_train[:200]))

# %%
sc.sum(sc.round(sc.clip(results.labels * y_train[:200], 0, 1)))

# %%
sc.sum(sc.round(sc.clip(y_train[:200], 0, 1)))

# %%
sc.statistics.mean(y_train[:400] == results.labels)

max_stars_repo_name: shapelets/shapelets-compute

max_issues_repo_name: shapelets/shapelets-compute

max_forks_repo_name: shapelets/shapelets-compute