text:
# -*- coding: utf-8 -*-
"""Contains classes and functions related to Algorithms for Enumerating All Perfect,
Maximum and Maximal Matchings in Bipartite Graphs. From Takeaki Uno publication.

The function `enum_perfect_matchings` can be used to enumerate all maximum matchings of a `BipartiteGraph`.
The function `enum_maximum_matchings` can be used to enumerate all maximum matchings of a `BipartiteGraph`.
"""
from typing import cast, Iterator, Any, Dict, List, Tuple

import copy
import networkx as nx
from networkx.algorithms.bipartite.matching import maximum_matching

from .graphs_utils import (create_directed_matching_graph, find_cycle_with_edge_of_matching,
                           find_feasible_two_edge_path, graph_without_edge,
                           graph_without_nodes_of_edge,
                           strongly_connected_components_decomposition, top_nodes, bottom_nodes)

LEFT = 0
RIGHT = 1

__all__ = ['enum_perfect_matchings', 'enum_maximum_matchings', 'enum_maximal_matchings']

def enum_perfect_matchings(graph: nx.Graph) -> Iterator[Dict[Any, Any]]:
    if len(list(top_nodes(graph))) != len(list(bottom_nodes(graph))):
        return
    size = len(list(top_nodes(graph)))
    matching = maximum_matching(graph, top_nodes=top_nodes(graph))
    # Express the matching only from a top node to a bottom node
    matching = {k: v for k, v in matching.items() if k in top_nodes(graph)}
    if matching and len(matching) == size:
        yield matching
        directed_match_graph = create_directed_matching_graph(graph, top_nodes(graph), matching)
        trimmed_directed_match_graph = strongly_connected_components_decomposition(
            directed_match_graph)
        graph = trimmed_directed_match_graph.to_undirected()
        assert len(graph.edges) == len(trimmed_directed_match_graph.edges)
        assert len(graph.nodes) == len(trimmed_directed_match_graph.nodes)
        yield from _enum_perfect_matchings_iter(graph=copy.deepcopy(graph), matching=matching)

def _start_cycle_with_left(graph: nx.Graph, raw_cycle: List[Any]) -> Tuple[Any, ...]:
    # Make sure the cycle "starts"" in the the left part
    # If not, start the cycle from the second node, which is in the left part
    if graph.nodes[raw_cycle[0]]['bipartite'] != LEFT:
        return tuple(raw_cycle[-1:] + raw_cycle[:-1])
    return tuple(raw_cycle[:])

def _enum_perfect_matchings_iter(graph: nx.Graph, matching: Dict[Any,
                                                                 Any]) -> Iterator[Dict[Any, Any]]:
    # Algorithm described in "Algorithms for Enumerating All Perfect, Maximum and Maximal Matchings in Bipartite Graphs"
    # By Takeaki Uno in "Algorithms and Computation: 8th International Symposium, ISAAC '97 Singapore,
    # December 17-19, 1997 Proceedings"
    # See http://dx.doi.org/10.1007/3-540-63890-3_11

    # Step 1
    if len(graph.edges) == 0:
        return

    # Find a cycle in the directed matching graph
    # Note that this cycle alternates between nodes from the left and the right part of the graph

    # TODO: avoid doing a directed graph by implementing missing algorithm steps from paper
    directed_match_graph = create_directed_matching_graph(graph, top_nodes(graph), matching)

    try:
        raw_cycle = find_cycle_with_edge_of_matching(graph=directed_match_graph, matching=matching)
    except nx.NetworkXNoCycle:
        return

    cycle = _start_cycle_with_left(graph, raw_cycle)
    assert directed_match_graph.nodes[cycle[0]]['bipartite'] == LEFT

    # Step 2 - TODO: Properly find right edge? (to get complexity bound)
    edge = cast(Tuple[Any, Any], tuple(cycle[:2]))

    # Step 3
    # already done because we are not really finding the optimal edge

    # Step 4
    # Construct new matching M' by flipping edges along the cycle, i.e. change the direction of all the
    # edges in the cycle
    matching_prime = matching.copy()
    for i in range(0, len(cycle), 2):
        matching_prime[cycle[i]] = cycle[i - 1]
    assert matching_prime != matching
    yield matching_prime

    # Construct G+(e)
    graph_plus = graph_without_nodes_of_edge(graph, edge)

    # Step 5
    # Trim unnecessary edges from G+(e).
    directed_match_graph_plus = create_directed_matching_graph(graph_plus, top_nodes(graph_plus),
                                                               matching)
    trimmed_directed_match_graph_plus = strongly_connected_components_decomposition(
        directed_match_graph_plus)
    graph_plus = trimmed_directed_match_graph_plus.to_undirected()
    assert len(graph_plus.edges) == len(trimmed_directed_match_graph_plus.edges)
    assert len(graph_plus.nodes) == len(trimmed_directed_match_graph_plus.nodes)

    # Step 6
    # Recurse with the old matching M but without the edge e
    yield from _enum_perfect_matchings_iter(graph_plus, matching)

    # Construct G-(e)
    graph_minus = graph_without_edge(graph, edge)

    # Step 7
    # Trim unnecessary edges from G-(e).
    directed_match_graph_minus = create_directed_matching_graph(graph_minus,
                                                                top_nodes(graph_minus),
                                                                matching_prime)
    trimmed_directed_match_graph_minus = strongly_connected_components_decomposition(
        directed_match_graph_minus)
    graph_minus = trimmed_directed_match_graph_minus.to_undirected()
    assert len(graph_minus.edges) == len(trimmed_directed_match_graph_minus.edges)
    assert len(graph_minus.nodes) == len(trimmed_directed_match_graph_minus.nodes)

    # Step 8
    # Recurse with the new matching M' but without the edge e
    yield from _enum_perfect_matchings_iter(graph_minus, matching_prime)

def enum_maximum_matchings(graph: nx.Graph) -> Iterator[Dict[Any, Any]]:
    matching = maximum_matching(graph, top_nodes=top_nodes(graph))
    # Express the matching only from a top node to a bottom node
    matching = {k: v for k, v in matching.items() if k in top_nodes(graph)}
    if matching:
        yield matching
        directed_match_graph = create_directed_matching_graph(graph, top_nodes(graph), matching)
        trimmed_directed_match_graph = strongly_connected_components_decomposition(
            directed_match_graph)
        yield from _enum_maximum_matchings_iter(graph=copy.deepcopy(graph),
                                                matching=matching,
                                                directed_match_graph=trimmed_directed_match_graph)

def _get_new_matching_by_exchanging_edges(matching: Dict[Any, Any],
                                          two_edge_path: Tuple[Any, Any, Any]) -> Dict[Any, Any]:
    (first, second, third) = two_edge_path

    # Construct M'
    if first in matching.keys():
        matching_prime = matching.copy()
        del matching_prime[first]
        matching_prime[third] = second
    else:
        matching_prime = matching.copy()
        del matching_prime[second]
        matching_prime[second] = third
    return matching_prime

def _enum_maximum_matchings_iter(graph: nx.Graph, matching: Dict[Any, Any],
                                          directed_match_graph: nx.DiGraph) \
        -> Iterator[Dict[Any, Any]]:
    # Algorithm described in "Algorithms for Enumerating All Perfect, Maximum and Maximal Matchings in Bipartite Graphs"
    # By Takeaki Uno in "Algorithms and Computation: 8th International Symposium, ISAAC '97 Singapore,
    # December 17-19, 1997 Proceedings"
    # See http://dx.doi.org/10.1007/3-540-63890-3_11

    # Step 1
    if len(graph.edges) == 0:
        return

    # Step 2
    # Find a cycle in the directed matching graph
    # Note that this cycle alternates between nodes from the left and the right part of the graph
    try:
        raw_cycle = find_cycle_with_edge_of_matching(graph=directed_match_graph, matching=matching)
    except nx.NetworkXNoCycle:
        raw_cycle = []

    if raw_cycle:
        cycle = _start_cycle_with_left(graph, raw_cycle)
        assert directed_match_graph.nodes[cycle[0]]['bipartite'] == LEFT

        # Step 3 - TODO: Properly find right edge? (to get complexity bound)
        edge = cast(Tuple[Any, Any], tuple(cycle[:2]))

        # Step 4
        # already done because we are not really finding the optimal edge

        # Step 5
        # Construct new matching M' by flipping edges along the cycle, i.e. change the direction of all the
        # edges in the cycle
        matching_prime = matching.copy()
        for i in range(0, len(cycle), 2):
            matching_prime[cycle[i]] = cycle[i - 1]

        assert matching_prime != matching
        yield matching_prime

        # Step 6
        # Construct G+(e) and D(G+(e), M\e)
        graph_plus = graph_without_nodes_of_edge(graph, edge)
        directed_match_graph_plus = create_directed_matching_graph(graph_plus,
                                                                   top_nodes(graph_plus), matching)
        # Recurse with the old matching M but without the edge e
        yield from _enum_maximum_matchings_iter(graph_plus, matching, directed_match_graph_plus)

        # Step 7
        # Construct G-(e) and D(G-(e), M')
        graph_minus = graph_without_edge(graph, edge)
        directed_match_graph_minus = create_directed_matching_graph(graph_minus,
                                                                    top_nodes(graph_minus),
                                                                    matching_prime)
        # Recurse with the new matching M' but without the edge e
        yield from _enum_maximum_matchings_iter(graph_minus, matching_prime,
                                                directed_match_graph_minus)

    else:
        # Step 8
        # Find feasible path of length 2 in D(graph, matching)
        two_edge_path = find_feasible_two_edge_path(graph, matching)
        if not two_edge_path:
            return

        # remove the node that is in the matching to only keep the new edge
        edge = two_edge_path[:0:-1] if two_edge_path[0] in matching.keys() else two_edge_path[1::]
        matching_prime = _get_new_matching_by_exchanging_edges(matching=matching,
                                                               two_edge_path=two_edge_path)
        assert matching_prime != matching
        yield matching_prime

        # Step 9
        # Construct G+(e) and D(G+(e), M\e)
        graph_plus = graph_without_nodes_of_edge(graph, edge)
        dgm_plus = create_directed_matching_graph(graph_plus, top_nodes(graph_plus),
                                                  matching_prime)
        yield from _enum_maximum_matchings_iter(graph_plus, matching_prime, dgm_plus)

        # Step 10
        # Construct G-(e) and D(G-(e), M')
        graph_minus = graph_without_edge(graph, edge)
        dgm_minus = create_directed_matching_graph(graph_minus, top_nodes(graph_minus), matching)
        yield from _enum_maximum_matchings_iter(graph_minus, matching, dgm_minus)

def enum_maximal_matchings(graph: nx.Graph) -> Iterator[Dict[Any, Any]]:
    # Step 1
    # If all vertices of G have degrees 0 or 1, output the unique maximal matching of G and stop.
    node_degree = next((node_degree for node_degree in graph.degree if node_degree[1] >= 2), None)
    if node_degree is None:
        matching = maximum_matching(graph, top_nodes=top_nodes(graph))
        yield {k: v for k, v in matching.items() if k in top_nodes(graph)}
        return

    # Step 2
    # Choose a vertex v with degree at least 2.
    node, degree = node_degree

    for neighbor in graph.neighbors(node):
        # Step 3: For each edge e in G incident to v, construct G+(e) and enumerate all maximal
        # matchings including e by recursive calls.
        # After the recursive call, reconstruct G from G+(e)

        # Oder nodes in the edge according to matching convention
        edge = (node, neighbor)
        edge = edge if edge[0] in top_nodes(graph) else edge[::-1]
        # Create G+(e)
        graph_plus = graph_without_nodes_of_edge(graph, edge)
        # Recursively get maximal matchings from G+(e) and then add e
        for maximal_matching in enum_maximal_matchings(graph_plus):
            maximal_matching.update({edge})
            yield maximal_matching

    # Let G' be the subgraph composed of edges incident
    # to vertices adjacent to v, except for edges incident to v
    edges = set()
    for neighbor in graph.neighbors(node):
        for second_neighbor in graph.neighbors(neighbor):
            if second_neighbor is not node:
                edges.add(tuple(sorted((neighbor, second_neighbor))))
    graph_prime = graph.edge_subgraph(edges)
    # Step 4
    # Find a maximum matching M in G'. If |M| = d(v),
    # then enumerate all maximum matchings in G' by ENUM_MAXIMUM_MATCHING_ITER(M,G').
    matching = maximum_matching(graph_prime, top_nodes=top_nodes(graph_prime))
    matching = {k: v for k, v in matching.items() if k in top_nodes(graph_prime)}
    if len(matching) == degree:
        directed_match_graph = create_directed_matching_graph(graph_prime, top_nodes(graph_prime),
                                                              matching)
        for max_matching in _enum_maximum_matchings_iter(graph_prime, matching,
                                                         directed_match_graph):
            # Step 5
            # For each matching, enumerate all maximal matchings including it.
            subgraph = nx.Graph(graph_prime)
            for edge in max_matching.items():
                subgraph = graph_without_nodes_of_edge(subgraph, edge)
            for maximal_matching in enum_maximal_matchings(subgraph):
                maximal_matching.update(max_matching)
                yield maximal_matching

max_stars_repo_name: FranciscoMoretti/PyBipartiteMatching

max_issues_repo_name: FranciscoMoretti/PyBipartiteMatching

reason: import networkx,from networkx


text:
import pandas as pd
import requests
from bs4 import BeautifulSoup
import numpy as np
from time import sleep
from random import randint

pd.set_option('display.max_rows', None)
pd.set_option('display.max_columns', None)
pd.set_option('display.width', None)
pd.set_option('display.max_colwidth', -1)

url1="https://www.imdb.com/search/title/?groups=top_1000&ref_=adv_prv"
response=requests.get(url1)
soup1=BeautifulSoup(response.content,"lxml")
body=soup1.select("div.lister.list.detail.sub-list")[0]
indivs=body.select("div.lister-item.mode-advanced")
name=[i.select("h3.lister-item-header")[0].get_text().strip().replace("\n","")[2:].replace(".","")[:-6].replace("(I)","").replace("(II)","").replace("(III)","") for i in indivs]
para=[i.select("p.text-muted")[1].get_text().strip() for i in indivs]
rating=[i.select("div.inline-block.ratings-imdb-rating strong")[0].get_text() for i in indivs]
pages=np.arange(51,1001,50)
para1=[]
name1=[]
rating1=[]
for i in pages:
    link="https://www.imdb.com/search/title/?groups=top_1000&start={i}&ref_=adv_nxt".format(i=i)
    response=requests.get(link)
    soup=BeautifulSoup(response.content,"lxml")
    body=soup.select("div.lister.list.detail.sub-list")[0]
    indivs=body.select("div.lister-item.mode-advanced")
    sleep(randint(2,6))
    for i in indivs:
        name1.append(i.select("h3.lister-item-header")[0].get_text().strip().replace("\n","")[3:].replace(".","")[:-6].replace("00","").replace("(I)","").replace("(II)","").replace("(III)",""))
        para1.append(i.select("p.text-muted")[1].get_text().strip())
        rating1.append(i.select("div.inline-block.ratings-imdb-rating strong")[0].get_text())
final_name=name+name1
final_para=para+para1
final_rating=rating+rating1
df=pd.DataFrame({"Movie Name":final_name,"IMDB Rating":final_rating,"Movie Summary":final_para},index=[indexx for indexx in range(1,len(final_name)+1)])
df.to_csv("imdb_full_list.csv")
df

ext: py

max_stars_repo_name: crispinsujith/Webscraping

max_issues_repo_name: crispinsujith/Webscraping

max_forks_repo_name: crispinsujith/Webscraping

reason: import numpy


text:
"""
====================
Jumeg MFT example.

Perform MFT on a surface based forward solution.
"""

import numpy as np
import os
import mne
from mne.datasets import sample
from jumeg.mft import apply_mft
from jumeg.mft import jumeg_mft_plot

data_path = sample.data_path()
subject = 'sample'
subjects_dir = data_path + '/subjects'
fwdname = data_path + '/MEG/sample/sample_audvis-meg-oct-6-fwd.fif'
lblname = 'aparc'
evoname = data_path + '/MEG/sample/sample_audvis-ave.fif'
evocondition = 'Left Auditory'
rawname = data_path + '/MEG/sample/sample_audvis_10s-raw.fif'
t1_fname = subjects_dir + '/' + 'sample/mri/T1.mgz'

# Set up pick list: MEG - bad channels
want_meg = 'mag'
want_ref = False
want_eeg = False
want_stim = False
exclude = 'bads'
include = []

print("########## MFT parameters:")
# mftpar = { 'prbfct':'Gauss',
#           'prbcnt':np.array([[-1.039, 0.013,0.062],[-0.039, 0.013,0.062]]),
#           'prbhw':np.array([[0.040,0.040,0.040],[0.040,0.040,0.040]]) }
mftpar = {'prbfct': 'uniform',
          'prbcnt': None,
          'prbhw': None}
mftpar.update({'iter': 2, 'currexp': 1.0})
mftpar.update({'regtype': 'PzetaE', 'zetareg': 1.00})
# mftpar.update({ 'regtype':'classic', 'zetareg':1.0})
mftpar.update({'solver': 'lu', 'svrelcut': 5.e-4})

print("mftpar['prbcnt'  ] = ", mftpar['prbcnt'])
print("mftpar['prbhw'   ] = ", mftpar['prbhw'])
print("mftpar['iter'    ] = ", mftpar['iter'])
print("mftpar['regtype' ] = ", mftpar['regtype'])
print("mftpar['zetareg' ] = ", mftpar['zetareg'])
print("mftpar['solver'  ] = ", mftpar['solver'])
print("mftpar['svrelcut'] = ", mftpar['svrelcut'])
cdmcut = 0.10
print("cdmcut = ", cdmcut)

print("########## get labels:")
if lblname is not None:
    labels = mne.read_labels_from_annot(subject, parc=lblname,
                                        subjects_dir=subjects_dir)
else:
    labels = None

print("##########################")
print("##### Calling apply_mft()")
print("##########################")
fwd = mne.read_forward_solution(fwdname, verbose=True)
fwdspec = mne.io.pick.pick_types_forward(fwd, meg=want_meg, ref_meg=False,
                                         eeg=False, exclude=exclude)
dataspec = mne.read_evokeds(evoname, condition=evocondition,
                            baseline=(None, 0), verbose=True)
fwdmag, qualmft, stc_mft = apply_mft(fwdspec, dataspec, evocondition=evocondition,
                                     subject=subject, meg=want_meg, save_stc=False,
                                     calccdm='all', cdmcut=cdmcut, cdmlabels=labels,
                                     mftpar=mftpar, verbose='verbose')

evo = mne.read_evokeds(evoname, condition=evocondition, baseline=(None, 0))
tmin = -0.2
tstep = 1. / evo.info['sfreq']

stcdata = stc_mft.data

print(" ")
print("########## Some geo-numbers:")
lhinds = np.where(fwdmag['source_rr'][:, 0] <= 0.)
rhinds = np.where(fwdmag['source_rr'][:, 0] > 0.)
print("> Discriminating lh/rh by sign of fwdmag['source_rr'][:,0]:")
print("> lhinds[0].shape[0] = ", lhinds[0].shape[0], " rhinds[0].shape[0] = ", rhinds[0].shape[0])
invmri_head_t = mne.transforms.invert_transform(fwdmag['info']['mri_head_t'])
mrsrc = np.zeros(fwdmag['source_rr'].shape)
mrsrc = mne.transforms.apply_trans(invmri_head_t['trans'], fwdmag['source_rr'], move=True)
lhmrinds = np.where(mrsrc[:, 0] <= 0.)
rhmrinds = np.where(mrsrc[:, 0] > 0.)
print("> Discriminating lh/rh by sign of fwdmag['source_rr'][:,0] in MR coords:")
print("> lhmrinds[0].shape[0] = ", lhmrinds[0].shape[0], " rhmrinds[0].shape[0] = ", rhmrinds[0].shape[0])

# plotting routines
jumeg_mft_plot.plot_global_cdv_dist(stcdata)
jumeg_mft_plot.plot_visualize_mft_sources(fwdmag, stcdata, tmin=tmin, tstep=tstep,
                                          subject=subject, subjects_dir=subjects_dir)
jumeg_mft_plot.plot_cdv_distribution(fwdmag, stcdata)
jumeg_mft_plot.plot_max_amplitude_data(fwdmag, stcdata, tmin=tmin, tstep=tstep,
                                       subject=subject)
jumeg_mft_plot.plot_max_cdv_data(stc_mft, lhmrinds, rhmrinds)
jumeg_mft_plot.plot_cdvsum_data(stc_mft, lhmrinds, rhmrinds)
jumeg_mft_plot.plot_quality_data(qualmft, stc_mft)
jumeg_mft_plot.plot_cdm_data(qualmft, stc_mft, cdmlabels=labels)
jumeg_mft_plot.plot_jlong_labeldata(qualmft, stc_mft, labels)
jumeg_mft_plot.plot_jtotal_labeldata(qualmft, stc_mft, labels)

jumeg_mft_plot.plot_jlong_data(qualmft, stc_mft)

print("Done.")

max_stars_repo_name: fboers/jumeg

max_issues_repo_name: fboers/jumeg

max_forks_repo_name: fboers/jumeg


text:
# Kornpob Bhirombhakdi
# kbhirombhakdi@stsci.edu

from polynomial2d.polynomial2d import Polynomial2D
from astropy.io import fits
import numpy as np

class Background:
    def __init__(self,gdata,norder,mdata=None,
                 rescale='default',
                 sigclip=(False,None,None),
                 container=None
                ):
        """
        Background is a class to facilitate background estimation. The routine uses Polynomial2D package (pip install polynomial2d).
        - gdata = data
        - norder = polynomial order
        - mdata = mask data with True for masking object
        - rescale, sigclip = parameters from Polynomial2D to perform rescaling of values (in order to improve the performance), and sigma clipping in iterative fitting process respectively.
            rescale = 'default' will set rescaling to be performed with method [-1,1] linear transformation of x1,x2,y.
            Note: Polynomial2D also uses rescalex.rescale.Rescale class (pip install rescalex) internally for rescaling.
        At the instantiation, self.poly2d is the Polynomial2D object.
        - Use self.poly2d.fit() to perform the background fitting.
        - Use self.poly2d.model['YFIT'] to access the estimated background.
        - Use self.poly2d.model['MASKFIT'] to access the final mask after iterations (with True for masked out data).
        Use method save(do_bkg,do_mask) to save outputs, given container.
        - Set do_bkg = True for saving self.poly2d.model['YFIT'] as ./savefolder/saveprefix_cutbkg.fits.
        - Set do_mask = True for saving self.poly2d.model['MASKFIT'] as ./savefolder/saveprefix_maskfit.fits.
        """
        self.gdata = gdata
        self.norder = norder
        self.container = container
        if mdata is None:
            self.mdata = np.full_like(gdata,False,dtype=bool)
        else: self.mdata = mdata
        if rescale == 'default':
            dor = True
            methodr = {'method':'linear','minmax':(-1.,1.)}
            self.rescale = (dor,methodr,methodr,methodr)
        else:
            self.rescale = rescale
        self.sigclip = sigclip
        self.poly2d = self._poly2d()
    def _poly2d(self):
        ny,nx = self.gdata.shape
        x1 = np.arange(nx)
        x2 = np.arange(ny)
        x1v,x2v = np.meshgrid(x1,x2)
        obj = Polynomial2D(x1=x1v,x2=x2v,y=self.gdata,mask=self.mdata,norder=self.norder,rescale=self.rescale,sigclip=self.sigclip)
        return obj
    ##########
    ##########
    ##########
    def save(self,do_bkg=True,do_maskfit=True):
        if self.container is None:
            raise ValueError('container must be specified to save')
        saveprefix = self.container.data['saveprefix']
        savefolder = self.container.data['savefolder']
        if do_yfit:
            phdu = fits.PrimaryHDU()
            imhdu = fits.ImageHDU()
            hdul = fits.HDUList([phdu,imhdu])
            hdul[1].data = self.poly2d.model['YFIT'].copy()
            string = './{0}/{1}_cutbkg.fits'.format(savefolder,saveprefix)
            hdul.writeto(string,overwrite=True)
            print('Save {0}'.format(string))
        if do_maskfit:
            phdu = fits.PrimaryHDU()
            imhdu = fits.ImageHDU()
            hdul = fits.HDUList([phdu,imhdu])
            hdul[1].data = self.poly2d.model['MASKFIT'].astype(int)
            string = './{0}/{1}_maskfit.fits'.format(savefolder,saveprefix)
            hdul.writeto(string,overwrite=True)
            print('Save {0}'.format(string))

max_stars_repo_name: bkornpob/hstphot

max_issues_repo_name: bkornpob/hstphot

max_forks_repo_name: bkornpob/hstphot

reason: import numpy,from astropy


text:
# -*- coding: utf-8 -*-
from pathlib import Path
from io import StringIO
import logging
from dotenv import find_dotenv, load_dotenv
import boto3
import json
import datetime
import gzip
import shutil
import re
import os
import io
import pandas as pd
import numpy as np

from sqlalchemy import create_engine

import preprocessor as p
from nltk.corpus import stopwords
import spacy
import nltk
from nltk.tokenize import TweetTokenizer
import string
from nltk import wordpunct_tokenize
from nltk.tokenize import word_tokenize
from nltk.stem import PorterStemmer

BUCKET = 'tweets-1301' # s3 bucket name

class TrainingDataMaker():

    '''Loads up the data for training the models, from Social Media for Public Health, downloaded to s3.
    This dataset contains contains annotations for whether a tweet is relevant to the topic of flu vaccination
    and if the author intends to receive a flu vaccine. Analysis of this dataset was published in:

    Xiaolei Huang, Michael C. Smith, Michael Paul, Dmytro Ryzhkov, Sandra Quinn, David Broniatowski, Mark Dredze. Examining Patterns of Influenza Vaccination in Social Media.
    AAAI Joint Workshop on Health Intelligence (W3PHIAI), 2017.

    Performs preprocessing.'''

    def __init__(self, logger):

        self.logger = logger
        self.s3_client = boto3.client('s3')
        self.raw_data_path = Path(__file__).resolve().parents[2].joinpath('data').resolve().joinpath('raw').resolve()
        self.interim_data_path = Path(__file__).resolve().parents[2].joinpath('data').resolve().joinpath('interim').resolve()
        self.processed_data_path = Path(__file__).resolve().parents[2].joinpath('data').resolve().joinpath('processed').resolve()

        oov_df = pd.read_csv(self.raw_data_path.joinpath('OOV_dict.csv')) # load the out of vocabulary file
        self.oov_informal = oov_df.informal.to_list()
        self.oov_formal = oov_df.translated.to_list()

        self.nlp = spacy.load("en_core_web_sm")
        self.ps = PorterStemmer()

        # tweet preprocessing setup ----------------------------------------
        p.set_options(p.OPT.URL, p.OPT.EMOJI, p.OPT.MENTION, p.OPT.RESERVED, p.OPT.SMILEY, p.OPT.HASHTAG) # get rid of everything but numbers

    def get_training_data(self):

        '''Reads in gzip JSON from s3, saves JSON locally.'''
        obj = self.s3_client.get_object(Bucket=BUCKET, Key='flu_vaccine.json_.gz')
        obj = io.BytesIO(obj['Body'].read())
        fname = self.raw_data_path.joinpath('data.json').resolve()

        with gzip.open(obj, 'rb') as f_in:
            with open(fname, 'wb') as f_out:
                shutil.copyfileobj(f_in, f_out)
                f_out.close()

        self.logger.info('loaded training data, now cleaning...')

    def get_covid19_vac_tweets(self):

        '''Queries the PostresSQL DB on AWS to get the COVID-19 incoming tweets.
        The DB is constantly streaming them in so for testing purposes here have put a limit for speed.

        '''

        db_string = "postgresql+psycopg2://postgres:" + POSTGRES_PASSWORD + "@database.cgiiehbwlxvq.us-east-2.rds.amazonaws.com"
        engine = create_engine(db_string)

        self.raw_cv19_df = pd.read_sql(
            "SELECT * FROM twitter.streamed_vaccine LIMIT 350000",
            con=engine
        )

    def clean_data(self):

        '''Turns JSON into prepped dataframe with needed info'''

        tweets = []
        for line in open(self.raw_data_path.joinpath('data.json').resolve(), 'r', encoding='utf8'):
            tweets.append(json.loads(line))

        data = [json.loads(line) for line in open(self.raw_data_path.joinpath('data.json').resolve(), 'r', encoding='utf8')]
        D = {
            'tweet_id': [],
            'tweet_text': [],
            'is_retweet': [],
            'FLU_vac_intend_to_receive': [],
            'FLU_vac_received': [],
            'FLU_vac_relevant': [],
            'FLU_vac_sentiment': []
        }

        for idx in range(0, len(data)):

            d = json.dumps(data[idx])
            loaded_d = json.loads(d)

            D['tweet_id'].append(loaded_d['id'])
            D['tweet_text'].append(loaded_d['tweet']['text'])
            D['is_retweet'].append(loaded_d['tweet']['retweeted'])
            try:
                D['FLU_vac_intend_to_receive'].append(loaded_d['label']["flu_vaccine_intent_to_receive"])
            except KeyError:
                D['FLU_vac_intend_to_receive'].append('not_available')
            try:
                D['FLU_vac_received'].append(loaded_d['label']["flu_vaccine_received"])
            except KeyError:
                D['FLU_vac_received'].append('not_available')
            try:
                D['FLU_vac_relevant'].append(loaded_d['label']["flu_vaccine_relevant"])
            except KeyError:
                D['FLU_vac_relevant'].append('not_available')
            try:
                D['FLU_vac_sentiment'].append(loaded_d['label']["flu_vaccine_sentiment"])
            except KeyError:
                D['FLU_vac_sentiment'].append('not_available')

        self.raw_df = pd.DataFrame.from_dict(D, orient='index').transpose()
        self.raw_df.to_csv(self.interim_data_path.joinpath('raw_df.csv').resolve())
        self.logger.info('handled JSON -> CSV conversion, saved locally...')

    def preprocess(self, t, lowercase=True):

        '''Initial removal of strange Tweet characters'''

        processed_tweet = p.clean(t)

        # Remove all the special characters
        processed_tweet = re.sub(r'\W', ' ', processed_tweet)

        # remove all single characters
        processed_tweet = re.sub(r'\s+[a-zA-Z]\s+', ' ', processed_tweet)

        # Remove single characters from the start
        processed_tweet = re.sub(r'\^[a-zA-Z]\s+', ' ', processed_tweet)

        # Substituting multiple spaces with single space
        processed_tweet= re.sub(r'\s+', ' ', processed_tweet, flags=re.I)

        # Removing prefixed 'b'
        processed_tweet = re.sub(r'^b\s+', '', processed_tweet)

        if lowercase:
            processed_tweet = processed_tweet.lower()
        return processed_tweet

    def token_process(self, raw_text):

        """
        Takes in a string of text, then performs the following:
        remove all punctuation, remove all stopwords, returns a list of the cleaned text
        """

        # Check characters to see if they are in punctuation
        nopunc = [char for char in list(raw_text) if char not in string.punctuation]
        # Join the characters again to form the string
        nopunc = ''.join(nopunc)
        # Now just remove any stopwords
        return [word for word in nopunc.lower().split() if word.lower() not in stopwords.words('english')]

    def replace(self, token):

        '''Helper function for removing out of vocabulary words'''

        replacement = None
        if token in self.oov_informal:
            replacement = self.oov_formal[self.oov_informal.index(token)]
        else:
            replacement = token
        return replacement

    def replace_OOV_words(self, text_token_list):

        '''
        Replace the informal tweet words like B2B, etc. with the human-annotated ones that are most likely to be correct.
        Documentation and OOV dictionary available in https://crisisnlp.qcri.org/lrec2016/lrec2016.html
        '''

        text_token_list_formal = [self.replace(t) for t in text_token_list]
        return text_token_list_formal

    def _gen_lemmas(self, tweet_col):
        # Parse the sentence using the loaded 'en' model object `nlp` using spacy
        sentence = ' '.join([c for c in tweet_col])
        doc = self.nlp(sentence)
        # Extract the lemma for each token and join
        return [token.lemma_ for token in doc]

    def _gen_stems(self, tweet_col):
        return [self.ps.stem(w) for w in tweet_col]

    # ------------------------------------- end Tweet processing setup

    def preprocess_tweet_df(self, df, tweet_text_col, lemma=False):

        '''Applies the tweet preprocessing functions in order'''

        out_df = df.copy()

        self.logger.info('preprocessing: removing emojis, strange characters...')
        out_df[tweet_text_col] = out_df[tweet_text_col].apply(str)
        out_df['tweet_clean'] = out_df.apply(lambda L: self.preprocess(L[tweet_text_col]), axis=1)
        self.logger.info('preprocessing: turning into tokens...')
        out_df['tweet_tokens'] = out_df.apply(lambda L: self.token_process(L.tweet_clean), axis=1)
        self.logger.info('preprocessing: translating out-of-vocabulary words...')
        out_df['tweet_tokens_formal'] = out_df.apply(lambda L: self.replace_OOV_words(L.tweet_tokens), axis=1)
        self.logger.info('preprocessing: generating stems...')
        out_df['tweet_stems_formal'] = out_df.apply(lambda L: self._gen_stems(L.tweet_tokens), axis=1)

        if lemma:
            self.logger.info('preprocessing: generating lemmas (this takes time)...')
            out_df['tweet_lemmas_formal'] = out_df.apply(lambda L: self._gen_lemmas(L.tweet_tokens_formal), axis=1)

        return out_df

    def execute_dataprep(self):

        #self.get_training_data()
        self.get_covid19_vac_tweets()
        self.clean_data()
        #self.training_clean_df = self.preprocess_tweet_df(self.raw_df, 'tweet_text', lemma=True)
        #self.training_clean_df.to_csv(self.processed_data_path.joinpath('training_nlp.csv'))
        self.cv19_clean_df = self.preprocess_tweet_df(self.raw_cv19_df, 'tweet_text', lemma=True)
        self.cv19_clean_df.to_csv(self.processed_data_path.joinpath('cv19_nlp.csv'))

def main():
    """ Runs data processing scripts to turn raw data from s3 into
        cleaned data ready to be analyzed (saved in ../processed).
    """
    logger = logging.getLogger(__name__)
    logger.info('making final data set from raw data')

    TrainingData = TrainingDataMaker(logger)
    TrainingData.execute_dataprep()

if __name__ == '__main__':
    log_fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
    logging.basicConfig(level=logging.INFO, format=log_fmt)

    # load up the .env entries as environment variables
    load_dotenv(find_dotenv())

    POSTGRES_PASSWORD = os.getenv("POSTGRES_PASSWORD")

    main()

max_stars_repo_name: RachelDoehr/nlp

max_issues_repo_name: RachelDoehr/nlp

max_forks_repo_name: RachelDoehr/nlp


text:
class Storage(object):
    """
    Store large data structures (e.g. numpy arrays) efficiently
    using joblib.

    Use:

    >>> from Storage import Storage
    >>> storage = Storage(cachedir='tmp_u01', verbose=1)
    >>> import numpy as np
    >>> a = np.linspace(0, 1, 100000) # large array
    >>> b = np.linspace(0, 1, 100000) # large array
    >>> storage.save('a', a)
    >>> storage.save('b', b)
    >>> # later
    >>> a = storage.retrieve('a')
    >>> b = storage.retrieve('b')
    """
    def __init__(self, cachedir='tmp', verbose=1):
        """
        Parameters
        ----------
        cachedir: str
             Name of directory where objects are stored in files.
        verbose: bool, int
             Let joblib and this class speak when storing files
             to disk.
        """
        import joblib
        self.memory = joblib.Memory(cachedir=cachedir,
                                    verbose=verbose)
        self.verbose = verbose
        self.retrieve = self.memory.cache(
            self.retrieve, ignore=['data'])
        self.save = self.retrieve

    def retrieve(self, name, data=None):
        if self.verbose > 0:
            print 'joblib save of', name
        return data


text:
#
# Copyright (c) 2019. Asutosh Nayak (nayak.asutosh@ymail.com)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#

"""
This is the crux of this application. This class holds the code for Q-Learning technique of Reinforcement learning.
Base class: PlayerBase
"""

import pickle
import numpy as np
import time

from Board import Board, NUM_CELLS, WIN_O, WIN_X, DRAW
from PlayerBase import PlayerBase

OUTPUT_PATH = ".\\output\\"
OUTPUT_FILE_NAME = "q_table.pickle"

WIN_VALUE = 1.0
DRAW_VALUE = 0.0
LOSE_VALUE = -1.0

MIN_Q_VALUE = -9999

class QTPlayer(PlayerBase):

    def __init__(self, side, alpha=0.9, gamma=0.95):
        self.q_table = {}  # type: Dict[string, [float]]. Stores QValues for all actions for each state of board.
        self.moves_history = []  # type: [(board_hash: int, move: int)]
        self.alpha = alpha
        self.gamma = gamma
        self.side = side
        super().__init__()

    def save_data(self):
        """ saves the q_table into 'output' folder of application."""
        pickle.dump(self.q_table, open(OUTPUT_PATH+OUTPUT_FILE_NAME+"_"+str(self.side), "wb"))
        print("output saved. Length of table=", len(self.q_table))

    def load_data(self, table: dict = None):
        """
        method can be used to load any existing trained model to continue training on it further
        :param table: type dict[int,[float]]
        :return: None
        """
        if table is None:
            try:
                self.q_table = pickle.load(open(OUTPUT_PATH+OUTPUT_FILE_NAME+"_"+str(self.side), "rb"))
                print("Table loaded from output file. Length=", len(self.q_table))
            except FileNotFoundError:
                print("Either provide a QTable dict to load or a path from where QTable dict can be loaded")
        else:
            self.q_table = table
            print("Table loaded from passed object")

    def make_move(self, board: Board) -> int:
        """ function to make a move based on the """
        board_hash = board.board_to_hash()
        q_vals = self.get_q_value(board_hash)
        # print("init q_vals", q_vals)
        check_count = 0
        while check_count < NUM_CELLS:
            move = np.argmax(q_vals)  # gives action/cell which has max Q-Value
            if board.is_move_valid(move):
                self.moves_history.append((board_hash, move))
                return move
            q_vals[move] = MIN_Q_VALUE
            check_count = check_count + 1

        # if no move found among q_vals, which shouldn't happen because 'make_move'
        # is called after checking if EMPTY cells are available
        print("Q-Values", q_vals, "\n", board_hash, "\n", str(board.board), check_count)
        raise Exception("No valid moves found")

    def get_q_value(self, board_hash: int):
        """
        Returns list of Q-Values for all possible actions for board state represented
        by hash value - board_hash
        """
        if board_hash not in self.q_table:
            # print("hash not found", board_hash)
            self.q_table[board_hash] = np.around(np.random.uniform(0.3, 0.7, 9), decimals=2)
            # np.full((NUM_CELLS,), self.q_init)
        return self.q_table[board_hash].copy()

    def update_q_table(self):
        pass

    def match_over(self, match_result: int):
        """
        Match is over. Update the Q-Learning Table as for all the moves in move history
        """
        stime = time.time()
        # print("Re-evaluating for result", match_result, "Length of move_history",len(self.moves_history))
        self.moves_history.reverse()
        final_val = -1  # final state should be WIN_VALUE if q_player won, else other values
        if match_result == self.side:
            final_val = WIN_VALUE
        elif match_result == DRAW:
            final_val = DRAW_VALUE
        else:
            final_val = LOSE_VALUE

        old_value = self.q_table[self.moves_history[0][0]][self.moves_history[0][1]]
        self.q_table[self.moves_history[0][0]][self.moves_history[0][1]] = \
            (1-self.alpha)*self.q_table[self.moves_history[0][0]][self.moves_history[0][1]] + self.alpha * final_val
        new_value = self.q_table[self.moves_history[0][0]][self.moves_history[0][1]]
        # print("Final=", final_val, "side=", self.side, "Move made=", self.moves_history[0][1], ", Old Value=",
        #      old_value, ", New Value=", new_value, "diff=", (new_value-old_value))
        for i, move in enumerate(self.moves_history[1:]):
            # Next board state of i^th move is (i-1)^th state, since we have reversed the moves history list.
            i_time = time.time()
            max_q_of_next_state = np.max(self.get_q_value(self.moves_history[i-1][0]))
            try:
                old_value = self.q_table[move[0]][move[1]]
                self.q_table[move[0]][move[1]] = (1-self.alpha)*self.q_table[move[0]][move[1]] + self.alpha*self.gamma\
                    * max_q_of_next_state + self.alpha*final_val
                new_value = self.q_table[move[0]][move[1]]
                # print("Final=", final_val, "side=", self.side, "Move made=", move[1], ", Old Value=", old_value,
                #      ", New Value=", new_value, "diff=", (new_value-old_value))
                # print("iter time", time.time()-i_time)
            except KeyError:
                print("Problem move is", move, self.q_table[move[0]] if move[0] in self.q_table else "key doesn't exist"
                      , "total hashes", len(self.q_table), "index", i)
                raise Exception("Key not found")
        # print("Runtime for match_over of QPlayer", (time.time() - stime) // 60, "min", (time.time() - stime) % 60)

    def next_match(self):
        self.moves_history = []


text:
from typing import Dict, List, Union

import numpy as np

from jina.executors.crafters import BaseCrafter

class AlbumentationsCrafter(BaseCrafter):
    """Apply transforms from the Albumentations package to the image.

    This crafter allows you to apply any of the Albumenation's transforms to
    an image. You can also compose as many transforms as you would like inside
    a single object. For a full list of available transforms, visit
    `Albumentations's GitHub page <https://github.com/albumentations-team/albumentations/#list-of-augmentations>`_
    or see `the documentation <https://albumentations.ai/docs/api_reference/augmentations/transforms/>`_.

    .. attention::
        Albumentations' transforms were created for the purpose of image augmentation,
        where the transforms are applied randomly. However in indexing/search
        context you will want the transforms to be applied deterministically.

        For that reason pay attention to the parameters and set them in a way that
        any randomness is removed. For example, if you want to rotate the image 45
        degrees, set the ``limit`` parameter in the ``Rotate`` transformation to
        ``[45,45]``. And you don't want to use some inherently random transforms,
        such as ``RandomRotate90``.

        We take care of some of this automatically: we set ``always_apply=True`` for
        all transforms, so that they are always applied.

    Example:
        The transforms that you provide will be applied in the order provided using
        ``A.Compose``. For example, the ``crafter`` in the example below

        .. code-block:: python

            from jina.hub.crafters.image.AlbumentationsCrafter import AlbumentationsCrafter

            transforms = [
                'HorizontalFlip',
                {'Rotate': {'limit': [45, 45]}}
            ]
            crafter = AlbumentationsCrafter(transforms)

        will have the equivalent effect as the ``transform`` below

        .. code-block:: python

            import albumentations as A

            transform = A.Compose([
                A.HorizontalFlip(always_apply=True),
                A.Rotate(limit=[45,45], always_apply=True)
            ])

    :param transforms: A list of transformations that should be applied.
        Each item in the list should be of the form
        ``{'TransformClass': kwargs}`` or ``'TransformClass'``.
    :param args:  Additional positional arguments
    :param kwargs: A dictionary with keyword arguments that will
        be passed to ``TransformClass`` at initialization
        (or an empty dict if there are no such arguments).
        The argument ``always_apply=True`` will be added to all kwargs
        automatically.
    """

    def __init__(
        self,
        transforms: List[Union[str, Dict[str, Dict]]] = ['HorizontalFlip'],
        *args,
        **kwargs
    ):
        """Set constructor."""
        super().__init__(*args, **kwargs)

        if not isinstance(transforms, list):
            self.logger.error('The `transform` argument has to be a list.')
            raise ValueError

        self.transforms_specification = transforms

    def post_init(self):
        import albumentations as A

        transforms_list = []
        for transform in self.transforms_specification:

            if isinstance(transform, str):
                tr_name, tr_kwargs = transform, {}
            elif isinstance(transform, dict):
                tr_name, tr_kwargs = next(iter(transform.items()))
            else:
                self.logger.error(
                    'Each element of the `transform` has to be either a dict'
                    ' or a string.'
                )
                raise ValueError

            tr_kwargs.update({'always_apply': True})

            try:
                tr_class = getattr(A, tr_name)
            except:
                self.logger.error(
                    f'The albumentations class `{tr_name}` does not seem to exist,'
                    ' check that you have not made a typo.'
                )
                raise ValueError

            try:
                alb_transform = tr_class(**tr_kwargs)
            except:
                self.logger.error(
                    f'Error instantiating albumentations class `{tr_name}`,'
                    ' check that all kwargs are valid.'
                )
                raise ValueError

            transforms_list.append(alb_transform)

        self.transforms = A.Compose(transforms_list)

    def craft(self, blob: 'np.ndarray', *args, **kwargs) -> 'np.ndarray':
        """Apply transformations to the image.

        :param blob: The image to transform, should be in ``[H, W, C]`` format,
            where ``C`` is the color channel, which is either RGB (siz 3), or BW
            (size 1).
        :return: The transformed image
        """

        return self.transforms(image=blob)['image']

max_stars_repo_name: akurniawan/jina-hub

max_issues_repo_name: akurniawan/jina-hub

max_forks_repo_name: akurniawan/jina-hub


text:
# Copyright (c) 2020, NVIDIA CORPORATION.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import cudf
import dask_cudf
import numpy as np
import bisect

class NumberMap:
    """
    Class used to translate external vertex ids to internal vertex ids
    in the cuGraph framework.

    Internal vertex ids are assigned by hashing the external vertex ids
    into a structure to eliminate duplicates, and the resulting list
    of unique vertices are assigned integers from [0, V) where V is
    the number of unique vertices.

    In Single GPU mode, internal vertex ids are constructed using
    cudf functions, with a cudf.DataFrame containing the mapping
    from external vertex identifiers and internal vertex identifiers
    allowing for mapping vertex identifiers in either direction.  In
    this mode, the order of the output from the mapping functions is
    non-deterministic.  cudf makes no guarantees about order.  If
    matching the input order is required set the preserve_order
    to True.

    In Multi GPU mode, internal vertex ids are constucted using
    dask_cudf functions, with a dask_cudf.DataFrame containing
    the mapping from external vertex identifiers and internal
    vertex identifiers allowing for mapping vertex identifiers
    in either direction.  In this mode, the partitioning of
    the number_map and the output from any of the mapping functions
    are non-deterministic.  dask_cudf makes no guarantees about the
    partitioning or order of the output.  As of this release,
    there is no mechanism for controlling that, this will be
    addressed at some point.
    """

    class SingleGPU:
        def __init__(self, df, src_col_names, dst_col_names, id_type,
                     store_transposed):
            self.col_names = NumberMap.compute_vals(src_col_names)
            self.df = cudf.DataFrame()
            self.id_type = id_type
            self.store_transposed = store_transposed

            source_count = 0
            dest_count = 0

            if store_transposed:
                dest_count = 1
            else:
                source_count = 1

            tmp = (
                df[src_col_names]
                .assign(count=source_count)
                .groupby(src_col_names)
                .sum()
                .reset_index()
                .rename(
                    columns=dict(zip(src_col_names, self.col_names)),
                    copy=False,
                )
            )

            if dst_col_names is not None:
                tmp_dst = (
                    df[dst_col_names]
                    .assign(count=dest_count)
                    .groupby(dst_col_names)
                    .sum()
                    .reset_index()
                )
                for newname, oldname in zip(self.col_names, dst_col_names):
                    self.df[newname] = tmp[newname].append(tmp_dst[oldname])
                self.df['count'] = tmp['count'].append(tmp_dst['count'])
            else:
                for newname, oldname in zip(self.col_names, dst_col_names):
                    self.df[newname] = tmp[newname]
                self.df['count'] = tmp['count']

            self.numbered = False

        def compute(self):
            if not self.numbered:
                tmp = self.df.groupby(self.col_names).sum().sort_values(
                    'count', ascending=False
                ).reset_index().drop(columns='count')

                tmp["id"] = tmp.index.astype(self.id_type)
                self.df = tmp
                self.numbered = True

        def to_internal_vertex_id(self, df, col_names):
            tmp_df = df[col_names].rename(
                columns=dict(zip(col_names, self.col_names)), copy=False
            )
            index_name = NumberMap.generate_unused_column_name(df.columns)
            tmp_df[index_name] = tmp_df.index
            return (
                self.df.merge(tmp_df, on=self.col_names, how="right")
                .sort_values(index_name)
                .drop(columns=[index_name])
                .reset_index()["id"]
            )

        def add_internal_vertex_id(self, df, id_column_name, col_names,
                                   drop, preserve_order):
            ret = None

            if preserve_order:
                index_name = NumberMap.generate_unused_column_name(df.columns)
                tmp_df = df
                tmp_df[index_name] = tmp_df.index
            else:
                tmp_df = df

            if "id" in df.columns:
                id_name = NumberMap.generate_unused_column_name(tmp_df.columns)
                merge_df = self.df.rename(columns={"id": id_name}, copy=False)
            else:
                id_name = "id"
                merge_df = self.df

            if col_names is None:
                ret = merge_df.merge(tmp_df, on=self.col_names, how="right")
            elif col_names == self.col_names:
                ret = merge_df.merge(tmp_df, on=self.col_names, how="right")
            else:
                ret = (
                    merge_df.merge(
                        tmp_df,
                        right_on=col_names,
                        left_on=self.col_names,
                        how="right",
                    )
                    .drop(columns=self.col_names)
                )

            if drop:
                ret = ret.drop(columns=col_names)

            ret = ret.rename(
                columns={id_name: id_column_name}, copy=False
            )

            if preserve_order:
                ret = ret.sort_values(index_name).reset_index(drop=True)

            return ret

        def from_internal_vertex_id(
            self, df, internal_column_name, external_column_names
        ):
            tmp_df = self.df.merge(
                df,
                right_on=internal_column_name,
                left_on="id",
                how="right",
            )
            if internal_column_name != "id":
                tmp_df = tmp_df.drop(columns=["id"])
            if external_column_names is None:
                return tmp_df
            else:
                return tmp_df.rename(
                    columns=dict(zip(self.col_names, external_column_names)),
                    copy=False,
                )

    class MultiGPU:
        def extract_vertices(
            df, src_col_names, dst_col_names,
            internal_col_names, store_transposed
        ):
            source_count = 0
            dest_count = 0

            if store_transposed:
                dest_count = 1
            else:
                source_count = 1

            s = (
                df[src_col_names]
                .assign(count=source_count)
                .groupby(src_col_names)
                .sum()
                .reset_index()
                .rename(
                    columns=dict(zip(src_col_names, internal_col_names)),
                    copy=False,
                )
            )
            d = None

            if dst_col_names is not None:
                d = (
                    df[dst_col_names]
                    .assign(count=dest_count)
                    .groupby(dst_col_names)
                    .sum()
                    .reset_index()
                    .rename(
                        columns=dict(zip(dst_col_names, internal_col_names)),
                        copy=False,
                    )
                )

            reply = cudf.DataFrame()

            for i in internal_col_names:
                if d is None:
                    reply[i] = s[i]
                else:
                    reply[i] = s[i].append(d[i])

            reply['count'] = s['count'].append(d['count'])

            return reply

        def __init__(
            self, ddf, src_col_names, dst_col_names, id_type, store_transposed
        ):
            self.col_names = NumberMap.compute_vals(src_col_names)
            self.val_types = NumberMap.compute_vals_types(ddf, src_col_names)
            self.val_types["count"] = np.int32
            self.id_type = id_type
            self.store_transposed = store_transposed
            self.ddf = ddf.map_partitions(
                NumberMap.MultiGPU.extract_vertices,
                src_col_names,
                dst_col_names,
                self.col_names,
                store_transposed,
                meta=self.val_types,
            )
            self.numbered = False

        # Function to compute partitions based on known divisions of the
        # hash value
        def compute_partition(df, divisions):
            sample = df.index[0]
            partition_id = bisect.bisect_right(divisions, sample) - 1
            return df.assign(partition=partition_id)

        def assign_internal_identifiers_kernel(
            local_id, partition, global_id, base_addresses
        ):
            for i in range(len(local_id)):
                global_id[i] = local_id[i] + base_addresses[partition[i]]

        def assign_internal_identifiers(df, base_addresses, id_type):
            df = df.assign(local_id=df.index.astype(np.int64))
            df = df.apply_rows(
                NumberMap.MultiGPU.assign_internal_identifiers_kernel,
                incols=["local_id", "partition"],
                outcols={"global_id": id_type},
                kwargs={"base_addresses": base_addresses},
            )

            return df.drop(columns=["local_id", "hash", "partition"])

        def assign_global_id(self, ddf, base_addresses, val_types):
            val_types["global_id"] = self.id_type
            del val_types["hash"]
            del val_types["partition"]

            ddf = ddf.map_partitions(
                lambda df: NumberMap.MultiGPU.assign_internal_identifiers(
                    df, base_addresses, self.id_type
                ),
                meta=val_types,
            )
            return ddf

        def compute(self):
            if not self.numbered:
                val_types = self.val_types
                val_types["hash"] = np.int32

                vertices = self.ddf.map_partitions(
                    lambda df: df.assign(hash=df.hash_columns(self.col_names)),
                    meta=val_types,
                )

                # Redistribute the ddf based on the hash values
                rehashed = vertices.set_index("hash", drop=False)

                # Compute the local partition id (obsolete once
                #   https://github.com/dask/dask/issues/3707 is completed)
                val_types["partition"] = np.int32

                rehashed_with_partition_id = rehashed.map_partitions(
                    NumberMap.MultiGPU.compute_partition,
                    rehashed.divisions,
                    meta=val_types,
                )

                val_types.pop('count')

                numbering_map = rehashed_with_partition_id.map_partitions(
                    lambda df: df.groupby(
                        self.col_names + ["hash", "partition"]
                    ).sum()
                    .sort_values('count', ascending=False)
                    .reset_index()
                    .drop(columns='count'),
                    meta=val_types
                )

                #
                #  Compute base address for each partition
                #
                counts = numbering_map.map_partitions(
                    lambda df: df.groupby("partition").count()
                ).compute()["hash"].to_pandas()
                base_addresses = np.zeros(len(counts) + 1, self.id_type)

                for i in range(len(counts)):
                    base_addresses[i + 1] = base_addresses[i] + counts[i]

                #
                #  Update each partition with the base address
                #
                numbering_map = self.assign_global_id(
                    numbering_map, cudf.Series(base_addresses), val_types
                )

                self.ddf = numbering_map.persist()
                self.numbered = True

        def to_internal_vertex_id(self, ddf, col_names):
            return self.ddf.merge(
                ddf,
                right_on=col_names,
                left_on=self.col_names,
                how="right",
            )["global_id"]

        def add_internal_vertex_id(self, ddf, id_column_name, col_names, drop,
                                   preserve_order):
            # At the moment, preserve_order cannot be done on
            # multi-GPU
            if preserve_order:
                raise Exception("preserve_order not supported for multi-GPU")

            ret = None
            if col_names is None:
                ret = self.ddf.merge(
                    ddf, on=self.col_names, how="right"
                )
            elif col_names == self.col_names:
                ret = self.ddf.merge(
                    ddf, on=col_names, how="right"
                )
            else:
                ret = self.ddf.merge(
                    ddf, right_on=col_names, left_on=self.col_names
                ).map_partitions(
                    lambda df: df.drop(columns=self.col_names)
                )

            if drop:
                ret = ret.map_partitions(lambda df: df.drop(columns=col_names))

            ret = ret.map_partitions(
                lambda df: df.rename(
                    columns={"global_id": id_column_name}, copy=False
                )
            )

            return ret

        def from_internal_vertex_id(
            self, df, internal_column_name, external_column_names
        ):
            tmp_df = self.ddf.merge(
                df,
                right_on=internal_column_name,
                left_on="global_id",
                how="right"
            ).map_partitions(lambda df: df.drop(columns="global_id"))

            if external_column_names is None:
                return tmp_df
            else:
                return tmp_df.map_partitions(
                    lambda df:
                    df.rename(
                        columns=dict(
                            zip(self.col_names, external_column_names)
                        ),
                        copy=False
                    )
                )

    def __init__(self, id_type=np.int32):
        self.implementation = None
        self.id_type = id_type

    def aggregate_count_and_partition(df):
        d = {}
        d['count'] = df['count'].sum()
        d['partition'] = df['partition'].min()
        return cudf.Series(d, index=['count', 'partition'])

    def compute_vals_types(df, column_names):
        """
        Helper function to compute internal column names and types
        """
        return {
            str(i): df[column_names[i]].dtype for i in range(len(column_names))
        }

    def generate_unused_column_name(column_names):
        """
        Helper function to generate an unused column name
        """
        name = 'x'
        while name in column_names:
            name = name + "x"

        return name

    def compute_vals(column_names):
        """
        Helper function to compute internal column names based on external
        column names
        """
        return [str(i) for i in range(len(column_names))]

    def from_dataframe(
            self, df, src_col_names, dst_col_names=None, store_transposed=False
    ):
        """
        Populate the numbering map with vertices from the specified
        columns of the provided DataFrame.

        Parameters
        ----------
        df : cudf.DataFrame or dask_cudf.DataFrame
            Contains a list of external vertex identifiers that will be
            numbered by the NumberMap class.
        src_col_names: list of strings
            This list of 1 or more strings contain the names
            of the columns that uniquely identify an external
            vertex identifier for source vertices
        dst_col_names: list of strings
            This list of 1 or more strings contain the names
            of the columns that uniquely identify an external
            vertex identifier for destination vertices
        store_transposed : bool
            Identify how the graph adjacency will be used.
            If True, the graph will be organized by destination.
            If False, the graph will be organized by source

        """
        if self.implementation is not None:
            raise Exception("NumberMap is already populated")

        if dst_col_names is not None and len(src_col_names) != len(
            dst_col_names
        ):
            raise Exception(
                "src_col_names must have same length as dst_col_names"
            )

        if type(df) is cudf.DataFrame:
            self.implementation = NumberMap.SingleGPU(
                df, src_col_names, dst_col_names, self.id_type,
                store_transposed
            )
        elif type(df) is dask_cudf.DataFrame:
            self.implementation = NumberMap.MultiGPU(
                df, src_col_names, dst_col_names, self.id_type,
                store_transposed
            )
        else:
            raise Exception("df must be cudf.DataFrame or dask_cudf.DataFrame")

        self.implementation.compute()

    def from_series(self, src_series, dst_series=None, store_transposed=False):
        """
        Populate the numbering map with vertices from the specified
        pair of series objects, one for the source and one for
        the destination

        Parameters
        ----------
        src_series: cudf.Series or dask_cudf.Series
            Contains a list of external vertex identifiers that will be
            numbered by the NumberMap class.
        dst_series: cudf.Series or dask_cudf.Series
            Contains a list of external vertex identifiers that will be
            numbered by the NumberMap class.
        store_transposed : bool
            Identify how the graph adjacency will be used.
            If True, the graph will be organized by destination.
            If False, the graph will be organized by source
        """
        if self.implementation is not None:
            raise Exception("NumberMap is already populated")

        if dst_series is not None and type(src_series) != type(dst_series):
            raise Exception("src_series and dst_series must have same type")

        if type(src_series) is cudf.Series:
            dst_series_list = None
            df = cudf.DataFrame()
            df["s"] = src_series
            if dst_series is not None:
                df["d"] = dst_series
                dst_series_list = ["d"]
            self.implementation = NumberMap.SingleGPU(
                df, ["s"], dst_series_list, self.id_type, store_transposed
            )
        elif type(src_series) is dask_cudf.Series:
            dst_series_list = None
            df = dask_cudf.DataFrame()
            df["s"] = src_series
            if dst_series is not None:
                df["d"] = dst_series
                dst_series_list = ["d"]
            self.implementation = NumberMap.MultiGPU(
                df, ["s"], dst_series_list, self.id_type, store_transposed
            )
        else:
            raise Exception(
                "src_series must be cudf.Series or " "dask_cudf.Series"
            )

        self.implementation.compute()

    def to_internal_vertex_id(self, df, col_names=None):
        """
        Given a collection of external vertex ids, return the internal
        vertex ids

        Parameters
        ----------
        df: cudf.DataFrame, cudf.Series, dask_cudf.DataFrame, dask_cudf.Series
            Contains a list of external vertex identifiers that will be
            converted into internal vertex identifiers

        col_names: (optional) list of strings
            This list of 1 or more strings contain the names
            of the columns that uniquely identify an external
            vertex identifier

        Returns
        ---------
        vertex_ids : cudf.Series or dask_cudf.Series
            The vertex identifiers.  Note that to_internal_vertex_id
            does not guarantee order or partitioning (in the case of
            dask_cudf) of vertex ids. If order matters use
            add_internal_vertex_id

        """
        tmp_df = None
        tmp_col_names = None
        if type(df) is cudf.Series:
            tmp_df = cudf.DataFrame()
            tmp_df["0"] = df
            tmp_col_names = ["0"]
        elif type(df) is dask_cudf.Series:
            tmp_df = dask_cudf.DataFrame()
            tmp_df["0"] = df
            tmp_col_names = ["0"]
        else:
            tmp_df = df
            tmp_col_names = col_names

        reply = self.implementation.to_internal_vertex_id(tmp_df,
                                                          tmp_col_names)

        if type(df) in [cudf.DataFrame, dask_cudf.DataFrame]:
            return reply["0"]
        else:
            return reply

    def add_internal_vertex_id(
        self, df, id_column_name="id", col_names=None, drop=False,
        preserve_order=False
    ):
        """
        Given a collection of external vertex ids, return the internal vertex
        ids combined with the input data.

        If a series-type input is provided then the series will be in a column
        named '0'. Otherwise the input column names in the DataFrame will be
        preserved.

        Parameters
        ----------
        df: cudf.DataFrame, cudf.Series, dask_cudf.DataFrame, dask_cudf.Series
            Contains a list of external vertex identifiers that will be
            converted into internal vertex identifiers

        id_column_name: (optional) string
            The name to be applied to the column containing the id
            (defaults to 'id')

        col_names: (optional) list of strings
            This list of 1 or more strings contain the names
            of the columns that uniquely identify an external
            vertex identifier

        drop: (optional) boolean
            If True, drop the column names specified in col_names from
            the returned DataFrame.  Defaults to False.

        preserve_order: (optional) boolean
            If True, do extra sorting work to preserve the order
            of the input DataFrame.  Defaults to False.

        Returns
        ---------
        df : cudf.DataFrame or dask_cudf.DataFrame
            A DataFrame containing the input data (DataFrame or series)
            with an additional column containing the internal vertex id.
            Note that there is no guarantee of the order or partitioning
            of elements in the returned DataFrame.

        """
        tmp_df = None
        tmp_col_names = None
        can_drop = True
        if type(df) is cudf.Series:
            tmp_df = df.to_frame("0")
            tmp_col_names = ["0"]
            can_drop = False
        elif type(df) is dask_cudf.Series:
            tmp_df = df.to_frame("0")
            tmp_col_names = ["0"]
            can_drop = False
        else:
            tmp_df = df

            if isinstance(col_names, list):
                tmp_col_names = col_names
            else:
                tmp_col_names = [col_names]

        return self.implementation.add_internal_vertex_id(
            tmp_df, id_column_name, tmp_col_names, (drop and can_drop),
            preserve_order
        )

    def from_internal_vertex_id(
        self,
        df,
        internal_column_name=None,
        external_column_names=None,
        drop=False,
    ):
        """
        Given a collection of internal vertex ids, return a DataFrame of
        the external vertex ids

        Parameters
        ----------
        df: cudf.DataFrame, cudf.Series, dask_cudf.DataFrame, dask_cudf.Series
            A list of internal vertex identifiers that will be
            converted into external vertex identifiers.  If df is a series type
            object it will be converted to a dataframe where the series is
            in a column labeled 'id'.  If df is a dataframe type object
            then internal_column_name should identify which column corresponds
            the the internal vertex id that should be converted

        internal_column_name: (optional) string
            Name of the column containing the internal vertex id.
            If df is a series then this parameter is ignored.  If df is
            a DataFrame this parameter is required.

        external_column_names: (optional) string or list of strings
            Name of the columns that define an external vertex id.
            If not specified, columns will be labeled '0', '1,', ..., 'n-1'

        drop: (optional) boolean
            If True the internal column name will be dropped from the
            DataFrame.  Defaults to False.

        Returns
        ---------
        df : cudf.DataFrame or dask_cudf.DataFrame
            The original DataFrame columns exist unmodified.  Columns
            are added to the DataFrame to identify the external vertex
            identifiers. If external_columns is specified, these names
            are used as the names of the output columns.  If external_columns
            is not specifed the columns are labeled '0', ... 'n-1' based on
            the number of columns identifying the external vertex identifiers.
        """
        tmp_df = None
        can_drop = True
        if type(df) is cudf.Series:
            tmp_df = df.to_frame("id")
            internal_column_name = "id"
            can_drop = False
        elif type(df) is dask_cudf.Series:
            tmp_df = df.to_frame("id")
            internal_column_name = "id"
            can_drop = False
        else:
            tmp_df = df

        output_df = self.implementation.from_internal_vertex_id(
            tmp_df, internal_column_name, external_column_names
        )

        if drop and can_drop:
            return output_df.drop(columns=internal_column_name)

        return output_df

    def column_names(self):
        """
        Return the list of internal column names

        Returns
        ----------
            List of column names ('0', '1', ..., 'n-1')
        """
        return self.implementation.col_names

    def renumber(df, src_col_names, dst_col_names, preserve_order=False,
                 store_transposed=False):
        """
        Given a single GPU or distributed DataFrame, use src_col_names and
        dst_col_names to identify the source vertex identifiers and destination
        vertex identifiers, respectively.

        Internal vertex identifiers will be created, numbering vertices as
        integers starting from 0.

        The function will return a DataFrame containing the original dataframe
        contents with a new column labeled 'src' containing the renumbered
        source vertices and a new column labeled 'dst' containing the
        renumbered dest vertices, along with a NumberMap object that contains
        the number map for the numbering that was used.

        Note that this function does not guarantee order in single GPU mode,
        and does not guarantee order or partitioning in multi-GPU mode.  If you
        wish to preserve ordering, add an index column to df and sort the
        return by that index column.

        Parameters
        ----------
        df: cudf.DataFrame or dask_cudf.DataFrame
            Contains a list of external vertex identifiers that will be
            numbered by the NumberMap class.
        src_col_names: string or list of strings
            This list of 1 or more strings contain the names
            of the columns that uniquely identify an external
            vertex identifier for source vertices
        dst_col_names: string or list of strings
            This list of 1 or more strings contain the names
            of the columns that uniquely identify an external
            vertex identifier for destination vertices
        store_transposed : bool
            Identify how the graph adjacency will be used.
            If True, the graph will be organized by destination.
            If False, the graph will be organized by source

        Returns
        ---------
        df : cudf.DataFrame or dask_cudf.DataFrame
            The original DataFrame columns exist unmodified.  Columns
            are added to the DataFrame to identify the external vertex
            identifiers. If external_columns is specified, these names
            are used as the names of the output columns.  If external_columns
            is not specifed the columns are labeled '0', ... 'n-1' based on
            the number of columns identifying the external vertex identifiers.

        number_map : NumberMap
            The number map object object that retains the mapping between
            internal vertex identifiers and external vertex identifiers.

        Examples
        --------
        >>> M = cudf.read_csv('datasets/karate.csv', delimiter=' ',
        >>>                   dtype=['int32', 'int32', 'float32'], header=None)
        >>>
        >>> df, number_map = NumberMap.renumber(df, '0', '1')
        >>>
        >>> G = cugraph.Graph()
        >>> G.from_cudf_edgelist(df, 'src', 'dst')
        """
        renumber_map = NumberMap()

        if isinstance(src_col_names, list):
            renumber_map.from_dataframe(df, src_col_names, dst_col_names)
            df = renumber_map.add_internal_vertex_id(
                df, "src", src_col_names, drop=True,
                preserve_order=preserve_order
            )
            df = renumber_map.add_internal_vertex_id(
                df, "dst", dst_col_names, drop=True,
                preserve_order=preserve_order
            )
        else:
            renumber_map.from_dataframe(df, [src_col_names], [dst_col_names])
            df = renumber_map.add_internal_vertex_id(
                df, "src", src_col_names, drop=True,
                preserve_order=preserve_order
            )

            df = renumber_map.add_internal_vertex_id(
                df, "dst", dst_col_names, drop=True,
                preserve_order=preserve_order
            )

        if type(df) is dask_cudf.DataFrame:
            df = df.persist()

        return df, renumber_map

    def unrenumber(self, df, column_name, preserve_order=False):
        """
        Given a DataFrame containing internal vertex ids in the identified
        column, replace this with external vertex ids.  If the renumbering
        is from a single column, the output dataframe will use the same
        name for the external vertex identifiers.  If the renumbering is from
        a multi-column input, the output columns will be labeled 0 through
        n-1 with a suffix of _column_name.

        Note that this function does not guarantee order or partitioning in
        multi-GPU mode.

        Parameters
        ----------
        df: cudf.DataFrame or dask_cudf.DataFrame
            A DataFrame containing internal vertex identifiers that will be
            converted into external vertex identifiers.

        column_name: string
            Name of the column containing the internal vertex id.

        preserve_order: (optional) bool
            If True, preserve the ourder of the rows in the output
            DataFrame to match the input DataFrame

        Returns
        ---------
        df : cudf.DataFrame or dask_cudf.DataFrame
            The original DataFrame columns exist unmodified.  The external
            vertex identifiers are added to the DataFrame, the internal
            vertex identifier column is removed from the dataframe.

        Examples
        --------
        >>> M = cudf.read_csv('datasets/karate.csv', delimiter=' ',
        >>>                   dtype=['int32', 'int32', 'float32'], header=None)
        >>>
        >>> df, number_map = NumberMap.renumber(df, '0', '1')
        >>>
        >>> G = cugraph.Graph()
        >>> G.from_cudf_edgelist(df, 'src', 'dst')
        >>>
        >>> pr = cugraph.pagerank(G, alpha = 0.85, max_iter = 500,
        >>>                       tol = 1.0e-05)
        >>>
        >>> pr = number_map.unrenumber(pr, 'vertex')
        >>>
        """
        if len(self.implementation.col_names) == 1:
            # Output will be renamed to match input
            mapping = {"0": column_name}
        else:
            # Output will be renamed to ${i}_${column_name}
            mapping = {}
            for nm in self.implementation.col_names:
                mapping[nm] = nm + "_" + column_name

        if preserve_order:
            index_name = NumberMap.generate_unused_column_name(df)
            df[index_name] = df.index

        df = self.from_internal_vertex_id(df, column_name, drop=True)

        if preserve_order:
            df = df.sort_values(
                index_name
            ).drop(index_name).reset_index(drop=True)

        if type(df) is dask_cudf.DataFrame:
            return df.map_partitions(
                lambda df: df.rename(columns=mapping, copy=False)
            )
        else:
            return df.rename(columns=mapping, copy=False)

max_stars_repo_name: divyegala/cugraph

max_issues_repo_name: divyegala/cugraph

max_forks_repo_name: divyegala/cugraph


text:
import numpy
from matchms import Spectrum
from matchms.filtering import add_fingerprint

def test_add_fingerprint_from_smiles():
    """Test if fingerprint it generated correctly."""
    spectrum_in = Spectrum(mz=numpy.array([], dtype="float"),
                           intensities=numpy.array([], dtype="float"),
                           metadata={"smiles": "[C+]#C[O-]"})

    spectrum = add_fingerprint(spectrum_in, nbits=16)
    expected_fingerprint = numpy.array([0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0])
    assert numpy.all(spectrum.get("fingerprint") == expected_fingerprint), "Expected different fingerprint."

def test_add_fingerprint_from_inchi():
    """Test if fingerprint it generated correctly."""
    spectrum_in = Spectrum(mz=numpy.array([], dtype="float"),
                           intensities=numpy.array([], dtype="float"),
                           metadata={"inchi": "InChI=1S/C2O/c1-2-3"})

    spectrum = add_fingerprint(spectrum_in, nbits=16)
    expected_fingerprint = numpy.array([0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0])
    assert numpy.all(spectrum.get("fingerprint") == expected_fingerprint), "Expected different fingerprint."

def test_add_fingerprint_no_smiles_no_inchi():
    """Test if fingerprint it generated correctly."""
    spectrum_in = Spectrum(mz=numpy.array([], dtype="float"),
                           intensities=numpy.array([], dtype="float"),
                           metadata={})

    spectrum = add_fingerprint(spectrum_in)
    assert spectrum.get("fingerprint", None) is None, "Expected None."

def test_add_fingerprint_empty_spectrum():
    """Test if empty spectrum is handled correctly."""

    spectrum = add_fingerprint(None)
    assert spectrum is None, "Expected None."


text:
import numpy as np
from PIL import Image
from tqdm import trange

def picDecrypt(file,password):
	# Image to array
	data = np.array(file)
	cache = np.zeros(data.shape, dtype=data.dtype)

	# height, width, a number used to generate new numbers, password
	h = data.shape[0]
	w = data.shape[1]
	global_passwd = password # Real password
	passwd = global_passwd
	u = 3.9999999

	# Calculate password for column decryption
	print('\nCalculating Password...')
	for x in trange(h):
	    chaos = [passwd]
	    for p in range(1, w+1):
	        chaos.append(u * chaos[p - 1] * (1 - chaos[p - 1]))
	    passwd = chaos.pop()
	    del chaos

	print('\nDecryping Column...')
	for x in trange(w):  # Column
	    chaos = [passwd]
	    mapping = {}
	    address = {}

	    for p in range(1, h+1):
	        chaos.append(u * chaos[p - 1] * (1 - chaos[p - 1]))
	    passwd = chaos.pop()

	    p = 0
	    for temp in chaos:
	        mapping[temp] = p
	        p += 1

	    chaos.sort()

	    p = 0
	    for temp in chaos:
	        address[mapping[temp]] = p
	        p += 1

	    del chaos
	    del mapping
	    for d in address.keys():
	        cache[address[d]][x] = data[d][x]
	    del address

	print('\nDecrypting Rows...')
	passwd = global_passwd
	for x in trange(h):  # Row
	    chaos = [passwd]
	    mapping = {}
	    address = {}

	    for p in range(1, w+1):
	        chaos.append(u * chaos[p - 1] * (1 - chaos[p - 1]))
	    passwd = chaos.pop()

	    p = 0
	    for temp in chaos:
	        mapping[temp] = p
	        p += 1

	    chaos.sort()

	    p = 0
	    for temp in chaos:
	        address[mapping[temp]] = p
	        p += 1

	    del chaos
	    del mapping
	    for d in address.keys():
	        data[x][address[d]] = cache[x][d]
	    del address

	# Array to image
	result = Image.fromarray(data)
	return result

if __name__ == '__main__':
	import time
	image = Image.open(input('Path：'))
	pword = float(input('Password(0-1)：'))
	if not 0<pword<1:raise ValueError
	save_path = input('Save to：')
	start_time = time.time()
	picDecrypt(image,pword).save(save_path)
	input('\nDecryption Finished...\nTime: {:.6f} second(s)\nPress any key...'.format(time.time()-start_time))

max_stars_repo_path: Useable Version/Decrypt.py

max_stars_repo_name: ShouNichi/PyPicEncypt

max_issues_repo_path: Useable Version/Decrypt.py

max_issues_repo_name: ShouNichi/PyPicEncypt

max_forks_repo_path: Useable Version/Decrypt.py

max_forks_repo_name: ShouNichi/PyPicEncypt


text:
# -*- coding: utf-8 -*-
"""
Created on Wed Aug  7 02:22:50 2019

@author: Ezone
"""

#from _future_ import print_function
import numpy as np
import matplotlib.pyplot as plt
from keras.models import Sequential
from keras.layers.core import Dense,Activation,Dropout
from keras.optimizers import SGD
from keras.utils import np_utils
import tensorflow as tf
import pandas as pd

np.random.seed(1671)

dataset = pd.read_csv("Train.csv")
X = dataset.iloc[:,0:14]
Y = dataset.iloc[:,-1].values

#print(dataset.info())
def f1(S):
    if S=='None':
        return 0
    else:
        return 1

X['is_holiday'] = X.is_holiday.apply(f1)

def weather_categorical(S):
    if S=='scattered clouds':
        return 1
    elif S=='broken clouds':
        return 2
    elif S=='overcast clouds':
        return 3
    elif S=='sky is clear':
        return 4
    elif S=='few clouds':
        return 5
    elif S=='light rain':
        return 6
    elif S=='light intensity drizzle':
        return 7
    elif S=='mist':
        return 8
    elif S=='haze':
        return 9
    elif S=='fog':
        return 10
    elif S=='proximity shower rain':
        return 11
    elif S=='drizzle':
        return 12
    elif S=='moderate rain':
        return 13
    elif S=='heavy intensity rain':
        return 14
    elif S=='proximity thunderstorm':
        return 15
    elif S=='thunderstorm with light rain':
        return 16
    elif S=='proximity thunderstorm with rain':
        return 17
    elif S=='heavy snow':
        return 18
    elif S=='heavy intensity drizzle':
        return 19
    elif S=='snow':
        return 20
    elif S=='thunderstorm with heavy rain':
        return 21
    elif S=='freezing rain':
        return 22
    elif S=='shower snow':
        return 23
    elif S=='light rain and snow':
        return 24
    elif S=='light intensity shower rain':
        return 25
    elif S=='SQUALLS':
        return 26
    elif S=='thunderstorm with rain':
        return 27
    elif S=='proximity thunderstorm with drizzle':
        return 28
    elif S=='thunderstorm':
        return 29
    elif S=='Sky is Clear':
        return 30
    elif S=='very heavy rain':
        return 31
    elif S=='thunderstorm with light drizzle':
        return 32
    elif S=='light snow':
        return 33
    elif S=='thunderstorm with drizzle':
        return 34
    elif S=='smoke':
        return 35
    elif S=='shower drizzle':
        return 36
    elif S=='light shower snow':
        return 37
    elif S=='sleeet':
        return 38
    else:
        return 4

X['weather_description'] = X.weather_description.apply(weather_categorical)

#del X['weather_type']
print(int(X['date_time'][0][11:13]))

#del X['snow_p_h']

def tym(S):
    return int(S[11:13])
X['date_time'] = X.date_time.apply(tym)

from sklearn.preprocessing import StandardScaler
sc_r = StandardScaler()
X=sc_r.fit_transform(x)
Y=sc_r.fit_transform(Y)

from sklearn.preprocessing import LabelEncoder, OneHotEncoder
labelencoder_X = LabelEncoder()
X['weather_type'] = labelencoder_X.fit_transform(X['weather_type'])

x=X
x=np.array(x)

onehotencoder = OneHotEncoder(categorical_features = [12])
x = onehotencoder.fit_transform(x).toarray()

x=np.delete(x,0,axis=1)

test_x=pd.read_csv("Test.csv")

test_x['is_holiday'] = test_x.is_holiday.apply(f1)
test_x['weather_description'] = test_x.weather_description.apply(weather_categorical)
test_x['date_time'] = test_x.date_time.apply(tym)

from sklearn.preprocessing import LabelEncoder, OneHotEncoder
labelencoder_X = LabelEncoder()
test_x['weather_type'] = labelencoder_X.fit_transform(test_x['weather_type'])

test_x=np.array(test_x)

onehotencoder = OneHotEncoder(categorical_features = [12])
test_x = onehotencoder.fit_transform(test_x).toarray()

from sklearn.ensemble import RandomForestRegressor

rf = RandomForestRegressor(n_estimators = 1000, random_state = 42)
rf.fit(x, Y);
predictions = rf.predict(test_x)

errors = abs(predictions - Y)
e=sum(errors)
s=sum(Y)
print(((s-e)/e)*100)

print('Mean Absolute Error:', round(np.mean(errors), 2), 'degrees.')

max_stars_repo_name: utkryuk/Traffic-Volume-Predictor

max_issues_repo_name: utkryuk/Traffic-Volume-Predictor

max_forks_repo_name: utkryuk/Traffic-Volume-Predictor