text:
import json
import os
from queue import Queue
import time
import networkx
import random
from helper import openURL

from .legacy.videos_channelid import channelVideo
from .config import MAXDEMANDS, REGIONCODE, YOUTUBE_SEARCH_URL, YOUTUBE_TOKEN   # to become a context.. #one_of_the_somedays..

class ChannelExtractor:

    # set the waiting time

    fullStructure = None

    @staticmethod
    def crawlChannel(graph,fullSchema,videoQueue,channelQueue):

        ChannelExtractor.fullStructure = fullSchema

        freshChannel = channelQueue.get()

        while True:
            # print("Abagain")

            ChannelExtractor.insertChannel(user=freshChannel,graph=graph)

            sv = channelVideo(freshchannel["channelId"], MAXDEMANDS , REGIONCODE , YOUTUBE_TOKEN)

            try:

                url_response = json.loads(openURL(YOUTUBE_SEARCH_URL, sv.params))
                nextPageToken = url_response.get("nextPageToken")
                sv.load_channel_videos(url_response)

                while nextPageToken:
                    sv.params.update({"pageToken": nextPageToken})
                    url_response = json.loads(openURL(YOUTUBE_SEARCH_URL, sv.params))
                    nextPageToken = url_response.get("nextPageToken")
                    sv.load_channel_videos(url_response)

                ChannelExtractor.scrapVideos(sv.videos,graph,videoQueue,freshchannel)

            except Exception as ex:

                print("\033[93m An exception has occured \030[90m")
                print(ex.with_traceback)

                #emptying the queue to channelinate the thread..
                while not channelQueue.empty():

                    channelQueue.task_done()

                    channelQueue.get()

            channelQueue.task_done()
            freshchannel = channelQueue.get()

    @staticmethod
    def insertChannel(freshChannel,graph):
        if freshChannel["channelId"] in graph :
            return
        graph.add_nodes_from(freshChannel["channelId"],[k for k in freshChannel.items()])

    @staticmethod
    def scrapVideos(vids,graph,videoQueue,freshchannel):
        for video in vids:
            if video["videoId"] not in graph: #just to put less noise in the queue
                videoQueue.add(video)

            graph.add_edge(video["videoId"],freshchannel["channelId"],other= "includes")


text:
import numpy as np
from pymatgen.core.operations import SymmOp
from sys import exit

data = np.load("wyckoff_symmetry.npy")
identity = data[1][0][0]
# loop sg's
for x in range(1, 231):
    seen = []
    # loop Wyckoff positions
    for i, y in enumerate(data[x]):
        # loop points
        for j, z in enumerate(y):
            if z not in seen:
                seen.append(z)
            else:
                if z != identity and z not in y:
                    print("Error. Seen:")
                    for x in seen:
                        print(x)
                    print("Current:")
                    print(z)
                    exit()
    print(x)

ext: py

max_stars_repo_name: ubikpt/PyXtal

max_issues_repo_name: ubikpt/PyXtal

max_forks_repo_name: ubikpt/PyXtal

reason: import numpy


text:
import numpy as np
from sysopt import Block, Metadata, Composite

class ConstVel2D(Block):
    # A block used for testing with constant velocity

    def __init__(self,vx=1,vy=1):
        self.vx=vx
        self.vy=vy
        metadata = Metadata(
            inputs=[],
            state=[],
            parameters=[],
            outputs=['dx','dy']
        )
        super().__init__(metadata)

    def compute_outputs(self, t, state, algebraic, inputs, parameters):
        vx = self.vx
        vy = self.vy
        return [vx, vy]

class PosCart(Block):
    """ Block to handle dynamics between position and velocity in arbitrary
    3-dimensional cartesion coordinates
    """

    def __init__(self,
                 s0 = 0,
                 s1 = 0,
                 s2 =0):
        metadata = Metadata(
            inputs=['v0','v1','v2'],
            state=['s0','s1','s2'],
            parameters=[],
            outputs=['s0','s1','s2']
        )
        super().__init__(metadata)

    def compute_dynamics(self, t, state, algebraic, inputs, parameters):
        v0, v1, v2 = inputs
        s0, s1, s2 = state

        dynamics = [
            v0,
            v1,
            v2,
        ]
        return dynamics

    def compute_outputs(self, t, state, algebraic, inputs, parameters):
        s0, s1, s2 = state
        return [s0, s1, s2]

class VelCart(Block):
    """ Block to handle Newtonian dynamics in abritrary -dimensional
    cartesion coordinates, assuming constant mass
    """

    def __init__(self,
                 v0 = 0,
                 v1 = 0,
                 v2 =0):
        metadata = Metadata(
            inputs=['F0','F1','F2'],
            state=['v0','v1','v2'],
            parameters=['m'],
            outputs=['v0','v1','v2']
        )
        super().__init__(metadata)

    def compute_dynamics(self, t, state, algebraic, inputs, parameters):
        F0, F1, F2 = inputs
        v0, v1, v2 = state

        dynamics = [
            F0/m,
            F1/m,
            F2/m,
        ]
        return dynamics

    def compute_outputs(self, t, state, algebraic, inputs, parameters):
        v0, v1, v2 = state
        return [v0, v1, v2]

max_stars_repo_name: andrewjlock/sysopt

max_issues_repo_name: andrewjlock/sysopt

max_forks_repo_name: andrewjlock/sysopt


text:
import sys, bisect, math, itertools, string, queue, copy
# import numpy as np
# import scipy
from collections import Counter,defaultdict,deque
from itertools import permutations, combinations
from heapq import heappop, heappush
from fractions import gcd

def main():
    n,k = map(int,input().split())
    a = list(map(int,input().split()))
    past = [0 for _ in range(n)]
    past[0] = 1
    root = [0]
    pre = 0
    while True:
        Next = a[pre]-1
        if past[Next]:
            break
        root.append(Next)
        pre = Next
        past[Next] = 1

    rev = []
    i = 0
    while root[len(root)-1-i] != Next:
        rev.append(root[len(root)-1-i])
        i += 1
    rev.reverse()
    rever = [Next]
    for i in range(len(rev)):
        rever.append(rev[i])
    n1 = len(root)
    n2 = len(rever)
    if n1-1 >= k:
        print(root[k]+1)
        return
    if rever[0] != 0:
        k -= n1-n2
    print(rever[(k)%n2] + 1)

if __name__ == "__main__":
    main()

reason: import numpy,import scipy


text:
#!/usr/bin/env python
"""qpOASES python distutils setup script."""

#
#  This file is part of qpOASES.
#
#  qpOASES -- An Implementation of the Online Active Set Strategy.
#  Copyright (C) 2007-2017 by Hans Joachim Ferreau, Andreas Potschka,
#  Christian Kirches et al. All rights reserved.
#
#  qpOASES is free software; you can redistribute it and/or
#  modify it under the terms of the GNU Lesser General Public
#  License as published by the Free Software Foundation; either
#  version 2.1 of the License, or (at your option) any later version.
#
#  qpOASES is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#  See the GNU Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public
#  License along with qpOASES; if not, write to the Free Software
#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#

#
#   Filename:  setup.py
#   Author:    Sebastian F. Walter, Manuel Kudruss (thanks to Felix Lenders)
#   Version:   3.2
#   Date:      2013-2017
#

import os
import numpy as np
import platform

from distutils.core import setup
from distutils.extension import Extension
from Cython.Distutils import build_ext
from Cython.Build import cythonize

BASEDIR = os.path.dirname(os.path.abspath(__file__))
BASEDIR = os.path.dirname(BASEDIR)
BASEDIR = os.path.dirname(BASEDIR)

extra_params = {}
extra_params['include_dirs'] = [
    '/usr/include',
    os.path.join(BASEDIR, 'include'),
    os.path.join(BASEDIR, 'include', 'qpOASES'),
    np.get_include()]
extra_params['extra_compile_args'] = ["-O2", "-Wno-unused-variable"]
extra_params['extra_link_args'] = ["-Wl,-O1", "-Wl,--as-needed"]

extra_params = extra_params.copy()
extra_params['libraries'] = ['qpOASES']

extra_params['library_dirs'] = ['/usr/lib', os.path.join(BASEDIR, 'bin')]
extra_params['language'] = 'c++'

if platform.system() in ['Linux', 'Darwin']:
    extra_params['extra_compile_args'] = ['-D__USE_LONG_INTEGERS__',
            '-D__USE_LONG_FINTS__']

if platform.system() == 'Darwin':
    extra_params['include_dirs'].append(
            '/Library/Developer/CommandLineTools/usr/include/c++/v1')
    extra_params['extra_compile_args'] += ['-stdlib=libc++',
        '-Wno-c++11-long-long']
    extra_params['extra_link_args'] = ['-stdlib=libc++'] # override the others!

if os.name == 'posix':
    extra_params['runtime_library_dirs'] = extra_params['library_dirs']
    # NOTE the python wrapper compiles the interface from the header files
    #      and therefore requires access to all defines made.
    #      Therefore, we provide the CPPFLAGS from the make_linux.mk file
    #      directly to the compile process by extra_compile_args.
    # NOTE not all FLAGS can be added automatically, e.g. DEF_SOLVER.
    #      Please fix this yourself in case of problems.
    # TODO maybe add automatic make file parsing and choose from those options
    extra_params['extra_compile_args'] += [
        "-Wall",
        "-pedantic",
        "-Wshadow",
        "-Wfloat-equal",
        "-O3",
        "-Wconversion",
        "-Wsign-conversion",
        "-finline-functions",
        "-fPIC",
        "-DLINUX",
        "-D__USE_LONG_INTEGERS__",
        "-D__USE_LONG_FINTS__",
        "-D__NO_COPYRIGHT__",
    ]

ext_modules = [
    Extension("qpoases",  ["qpoases.pyx", "qpoases.pxd"],   **extra_params),
]

setup(
    name='qpOASES interface',
    cmdclass={'build_ext': build_ext},
    ext_modules=cythonize(ext_modules),
)

max_forks_repo_name: robomechanics/spirit-software


text:
# BSD 3-Clause License; see https://github.com/scikit-hep/awkward-1.0/blob/master/LICENSE

from __future__ import absolute_import

import sys
import itertools

import numpy

import awkward1
import cupy

def test_cupy_refcount():
    o = cupy.arange(10)
    i = awkward1.layout.Index64.from_cupy(o)
    assert sys.getrefcount(o) == 3
    i2  = awkward1.layout.Index64.from_cupy(o)
    assert sys.getrefcount(o) == 4
    i3  = awkward1.layout.Index64.from_cupy(o)
    assert sys.getrefcount(o) == 5
    i4  = awkward1.layout.Index64.from_cupy(o)
    assert sys.getrefcount(o) == 6
    i5  = awkward1.layout.Index64.from_cupy(o)
    assert sys.getrefcount(o) == 7
    i6  = awkward1.layout.Index64.from_cupy(o)
    assert sys.getrefcount(o) == 8
    i7  = awkward1.layout.Index64.from_cupy(o)
    assert sys.getrefcount(o) == 9
    i8  = awkward1.layout.Index64.from_cupy(o)
    assert sys.getrefcount(o) == 10
    i9  = awkward1.layout.Index64.from_cupy(o)
    assert sys.getrefcount(o) == 11

    del(i9)
    assert sys.getrefcount(o) == 10
    del(i8)
    assert sys.getrefcount(o) == 9
    del(i7)
    assert sys.getrefcount(o) == 8
    del(i6)
    assert sys.getrefcount(o) == 7
    del(i5)
    assert sys.getrefcount(o) == 6
    del(i4)
    assert sys.getrefcount(o) == 5
    del(i3)
    assert sys.getrefcount(o) == 4
    del(i2)
    assert sys.getrefcount(o) == 3
    del(i)
    assert sys.getrefcount(o) == 2

reason: import numpy,import cupy


text:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from graphviz import Digraph

# Include this path if working in Google Colab
#d = '/content/gdrive/My Drive/p2-p-data-visualization-Pappaterra-Lucia/'
# If not
d = ''

# Colors
color_set = [plt.cm.tab20(0), plt.cm.tab20(2), plt.cm.tab20(4), plt.cm.tab20(6),
    plt.cm.tab20(8), plt.cm.tab20(10), plt.cm.tab20(12), plt.cm.tab20(14),
    plt.cm.tab20(16), plt.cm.tab20(18), plt.cm.tab20(1), plt.cm.tab20(3),
    plt.cm.tab20(5), plt.cm.tab20(7), plt.cm.tab20(9), plt.cm.tab20(11),
    plt.cm.tab20(13), plt.cm.tab20(15), plt.cm.tab20(17), plt.cm.tab20(19),
    plt.cm.tab20b(0), plt.cm.tab20b(2), plt.cm.tab20b(4), plt.cm.tab20b(6),
    plt.cm.tab20b(8), plt.cm.tab20b(10), plt.cm.tab20b(12), plt.cm.tab20b(14),
    plt.cm.tab20b(16), plt.cm.tab20b(18), plt.cm.tab20b(1), plt.cm.tab20b(3),
    plt.cm.tab20b(5), plt.cm.tab20b(7), plt.cm.tab20b(9), plt.cm.tab20b(11),
    plt.cm.tab20b(13), plt.cm.tab20b(15), plt.cm.tab20b(17), plt.cm.tab20b(19),
    plt.cm.Pastel1(0), plt.cm.Pastel1(1), plt.cm.Pastel1(2),
    plt.cm.Pastel1(3),plt.cm.Pastel1(4), plt.cm.Pastel1(5),
    plt.cm.Pastel1(6), plt.cm.Pastel1(7), plt.cm.Pastel1(8),
    plt.cm.Set3(0), plt.cm.Set3(1), plt.cm.Set3(2), plt.cm.Set3(3),
    plt.cm.Set3(4), plt.cm.Set3(5), plt.cm.Set3(6), plt.cm.Set3(7),
    plt.cm.Set3(8), plt.cm.Set3(9), plt.cm.Set3(10), plt.cm.Set3(11),
    plt.cm.Accent(7), plt.cm.Accent(6), plt.cm.Accent(5), plt.cm.Accent(4),
    plt.cm.Accent(3), plt.cm.Accent(2), plt.cm.Accent(1), plt.cm.Accent(0),
    plt.cm.Dark2(7), plt.cm.Dark2(6), plt.cm.Dark2(5), plt.cm.Dark2(4),
    plt.cm.Dark2(3), plt.cm.Dark2(2), plt.cm.Dark2(1), plt.cm.Dark2(0),
    plt.cm.tab20c(0), plt.cm.tab20c(2), plt.cm.tab20c(4), plt.cm.tab20c(6),
    plt.cm.tab20c(8), plt.cm.tab20c(10), plt.cm.tab20c(12), plt.cm.tab20c(14),
    plt.cm.tab20c(16), plt.cm.tab20c(18), plt.cm.tab20c(1), plt.cm.tab20c(3),
    plt.cm.tab20c(5), plt.cm.tab20c(7), plt.cm.tab20c(9), plt.cm.tab20c(11),
    plt.cm.tab20c(13), plt.cm.tab20c(15), plt.cm.tab20c(17), plt.cm.tab20c(19)]

def graph(df, df2):
    df['delivery_manager'].replace('', float("NaN"), inplace=True)
    df['delivery_manager'].fillna('No Delivery Managers yet', inplace=True)

    g = Digraph('OrgChart', filename='org_chart.gv')

    for p in df['project_name']:
        g.node(p, color='red')

        pm = df[df['project_name'] == p]['delivery_manager'].values[0]
        if pm=='':
            pm = []
        else:
            pm = pm.split(', ')

        rnr = list(df2[df2['project_name'] == p]['resource_name'])

        if len(rnr)>0:
            rnr = "\n".join(rnr)
            g.node(rnr, color='green', shape='rectangle')

            for elem in pm:
                g.node(elem, color='blue')
                g.edge(p, elem)
                if rnr is not None:
                    g.edge(elem, rnr)

    # save graph to pdf
    g.render(d+'outputs/teams_graph.gv', view=True)
    return g

def bar_plot(df, df2, colors=color_set):

    proj = list(df['project_name'])

    rnr = []
    for pr in proj:
        rnr.append(list(df2[df2['project_name']==pr]['resource_name']))

    rnr_counts=[]
    for elem in rnr:
        rnr_counts.append(len(elem))

    fig = plt.figure(figsize=(18,7))
    ax = fig.add_axes([0,0,1,1])
    langs = proj

    colors = tuple(colors[:len(proj)])

    ax.bar(langs, rnr_counts, color=colors)

    # Make some labels
    labels = [str(elem) for elem in rnr_counts]
    rects = ax.patches

    for rect, label in zip(rects, labels):
        height = rect.get_height()
        ax.text(rect.get_x() + rect.get_width() / 2, height, label,
                ha='center', va='bottom')

    plt.title('Amount of resources per project', fontsize=20)
    plt.xticks(rotation=90)
    plt.savefig(d+'outputs/bar_plot.png' , format='png', bbox_inches = 'tight')
    plt.show()

def pie_chart(df, df2, colors=color_set):
    df['delivery_manager'].replace('', float("NaN"), inplace=True)
    df['delivery_manager'].fillna('No Delivery Managers yet', inplace=True)

    proj = list(df['project_name'])

    rnr = []
    for p in proj:
        rnr.append(list(df2[df2['project_name']==p]['resource_name']))

    pms = convert_to_lists(df['delivery_manager'])

    rnr_size = []
    pms_size = []
    group_names = []

    for elem, elem2, elem3 in zip(rnr, pms, proj):
        n = len(elem)
        if n>0:
            rnr_size.append(n)
            m = len(elem2)
            pms_size.append(m)
            group_names.append(elem3)

    group_size = rnr_size

    subgroup_size = []
    for i, j in zip(rnr_size, pms_size):
        if j == 1:
            subgroup_size.append(i)
        else:
            for x in range(j):
                subgroup_size.append(i/j)

    fig, ax = plt.subplots(figsize=(12,7))
    ax.set_title("GlassWall Projects \nResources proportion", fontsize=20)

    ax.axis('equal')

    colors = colors[:len(proj)]

    colors1=[]
    pms2 = []
    for cr, i in zip(colors, range(len(rnr))):
        rs = rnr[i]
        if len(rs)>0:
            colors1.append(cr)
            pms2.append(pms[i])

    # First Ring (Inside)
    mypie, _ = ax.pie(group_size, radius=1-0.3, colors=colors1)
    plt.setp(mypie, width=0.4, edgecolor='white')

    colors2=[]
    for cr, pr, j in zip(colors1, group_names, range(len(pms2))):
        pm = pms2[j]
        if len(pm)>0:
            colors2 = colors2 + [cr] * len(pm)
        else:
            colors2 = colors2 + [cr]

    # Second Ring (Outside)
    mypie2, _ = ax.pie(subgroup_size, radius=1, colors=colors2)
    plt.setp(mypie2, width=0.4, edgecolor='white')

    ax.legend(mypie, group_names,
              title="Projects",
              loc="center left",
              bbox_to_anchor=(1, 0, 0.5, 1))

    bbox_props = dict(boxstyle="square,pad=0.3", fc="w", ec="k", lw=0.72)
    kw = dict(arrowprops=dict(arrowstyle="-"), bbox=bbox_props, zorder=0, va="center")

    total_res = sum(rnr_size)

    resources = []
    for elem in rnr_size:
        resources.append(str(elem) + ' resources ({d}%)'.format(d=round(elem/total_res*100)))

    for i, p in enumerate(mypie):
        ang = (p.theta2 - p.theta1)/2. + p.theta1
        y = np.sin(np.deg2rad(ang))
        x = np.cos(np.deg2rad(ang))
        horizontalalignment = {-1: "right", 1: "left"}[int(np.sign(x))]
        connectionstyle = "angle,angleA=0,angleB={}".format(ang)
        kw["arrowprops"].update({"connectionstyle": connectionstyle})
        ax.annotate(resources[i], xy=(x, y), xytext=(1.2*np.sign(x), 1.1*y),
            horizontalalignment=horizontalalignment, **kw)

    plt.margins(0,0)

    plt.savefig(d+'outputs/pie_chart.png' , format='png', bbox_inches = 'tight')

    # show it
    plt.show()

def convert_to_lists(df_col):
    ll=[]
    for elem in df_col:
        if elem=='':
            ll.append([])
        else:
            ll.append(elem.split(', '))
    return list(ll)


text:
# import the necessary packages
from .social_distancing_config import NMS_THRESH
from .social_distancing_config import MIN_CONF
import numpy as np
import cv2

def detect_people(frame, net, ln, personIdx=0):
	# grab the dimensions of the frame and  initialize the list of
	# results
	(H, W) = frame.shape[:2]
	results = []

	# construct a blob from the input frame and then perform a forward
	# pass of the YOLO object detector, giving us our bounding boxes
	# and associated probabilities
	blob = cv2.dnn.blobFromImage(frame, 1 / 255.0, (416, 416),
		swapRB=True, crop=False)
	net.setInput(blob)
	layerOutputs = net.forward(ln)

	# initialize our lists of detected bounding boxes, centroids, and
	# confidences, respectively
	boxes = []
	centroids = []
	confidences = []

	# loop over each of the layer outputs
	for output in layerOutputs:
		# loop over each of the detections
		for detection in output:
			# extract the class ID and confidence (i.e., probability)
			# of the current object detection
			scores = detection[5:]
			classID = np.argmax(scores)
			confidence = scores[classID]

			# filter detections by (1) ensuring that the object
			# detected was a person and (2) that the minimum
			# confidence is met
			if classID == personIdx and confidence > MIN_CONF:
				# scale the bounding box coordinates back relative to
				# the size of the image, keeping in mind that YOLO
				# actually returns the center (x, y)-coordinates of
				# the bounding box followed by the boxes' width and
				# height
				box = detection[0:4] * np.array([W, H, W, H])
				(centerX, centerY, width, height) = box.astype("int")

				# use the center (x, y)-coordinates to derive the top
				# and and left corner of the bounding box
				x = int(centerX - (width / 2))
				y = int(centerY - (height / 2))

				# update our list of bounding box coordinates,
				# centroids, and confidences
				boxes.append([x, y, int(width), int(height)])
				centroids.append((centerX, centerY))
				confidences.append(float(confidence))

	# apply non-maxima suppression to suppress weak, overlapping
	# bounding boxes
	idxs = cv2.dnn.NMSBoxes(boxes, confidences, MIN_CONF, NMS_THRESH)

	# ensure at least one detection exists
	if len(idxs) > 0:
		# loop over the indexes we are keeping
		for i in idxs.flatten():
			# extract the bounding box coordinates
			(x, y) = (boxes[i][0], boxes[i][1])
			(w, h) = (boxes[i][2], boxes[i][3])

			# update our results list to consist of the person
			# prediction probability, bounding box coordinates,
			# and the centroid
			r = (confidences[i], (x, y, x + w, y + h), centroids[i])
			results.append(r)

	# return the list of results
	return results


text:
# -*- coding: utf-8 -*-

'''
Tools for working with data from MLML public data portal:
http://pubdata.mlml.calstate.edu
'''

try:
    # For Python 3.0 and later
    from urllib.request import urlopen
except ImportError:
    # Fall back to Python 2's urllib2
    from urllib2 import urlopen
import re
import os
import sys
import numpy as np
from glob import glob
from datetime import datetime, timedelta
from physoce import util
try:
    import pandas as pd
except ImportError:
    pass
try:
    import xarray as xr
except ImportError:
    pass

def make_netcdf(station_dir,netcdf_file,station,download=False,overwrite=False):
    """
Create a netcdf file containing MLML historical seawater or weather data. The file will be created from csv and readme files already on disk, or they can be downloaded.

INPUT:
station_dir - string specifying the location of csv files (e.g. '/home/username/data/')
netcdf_file - string specifying the location and name of netcdf file to be created (e.g. '/home/username/data/mlml_seawater.nc')
station     - either 'seawater' or 'weather' (default: 'seawater')
download    - boolean specifying whether to download new files
              (default: False)
overwrite   - boolean specifying whether to overwrite the existing files, only used if downloading new data (default: False)
    """

    # download new data, if specified
    if download == True:
        download_station_data(station_dir,station,overwrite)

    # read data in csv files to xarray dataset
    d = read_csv_data(station_dir,format='dataset')

    # specify location of readme file and add metadata to dataset
    readme_file = station_dir + '1_README.TXT'
    _add_metadata_xarray(d,station,readme_file)
    d.attrs['history'] = d.attrs['history'] + 'netcdf file created using physoce.obs.mlml.make_netcdf(station_dir'+station_dir+',netcdf_file='+netcdf_file+',station='+station+'download='+str(download)+',overwrite='+str(overwrite)+'): ' + datetime.now().strftime("%Y-%m-%d %H:%M:%S") + ', '

    # create netcdf file
    d.to_netcdf(netcdf_file,mode='w')

def download_station_data(station_dir,station='seawater',overwrite=True):
    '''
Download all historical csv files for the MLML seawater intake or weather station. A latest version of the readme file is also downloaded. It is highly recommended to use different directories for seawater and weather, since the readme files have the same name. By default, new files are downloaded and existing files are overwritten.

INPUT:
station_dir - string specifying the local directory where you want to put
              the data files
station     - either 'seawater' or 'weather' (default: 'seawater')
overwrite   - boolean specifying whether to overwrite the existing files
              (default: False)

    '''
    # remote directories
    base_url = 'http://pubdata.mlml.calstate.edu/mlml_last/'
    station_url = base_url + '/' + station + '/'

    # local directory
    station_dir = station_dir + '/'

    # check whether a directory exists for this station
    if os.path.isdir(station_dir) == False:
        os.makedirs(station_dir)

    # find names of csv files that exist on the web and create a list
    # the csv filenames are in format yyyy-mm.csv
    urlpath =urlopen(station_url)
    html_string = urlpath.read().decode()
    urlpath.close()
    file_pattern = '[0-9][0-9][0-9][0-9]-[0-9][0-9].csv'
    csv_list = re.findall(file_pattern,html_string)

    # get updated readme file
    urlr = urlopen(station_url + '1_README.TXT')
    fr = open(station_dir + '1_README.TXT','w')
    fr.write(str(urlr.read()))
    fr.close()
    urlr.close()

    # loop through each remote csv file and download if:
    # the file does not exist, the overwrite option is True or it is the last
    # file in the list (there may be new data in that file)
    for csv_name in csv_list:
        print('downloading ' + station + ': ' + csv_name)
        remote_file = station_url + csv_name
        local_file = station_dir + csv_name
        write_conditions = [os.path.exists(local_file) == False,
                            overwrite == True,
                            csv_name == csv_list[-1]]
        if any(write_conditions):
            urlfile = urlopen(remote_file)
            f = open(local_file,'w')
            filebytes = urlfile.read()
            f.write(filebytes.decode('utf8'))
            f.close()
            urlfile.close()

def read_csv_data(data_dir,format='dict'):
    '''
Read historical text data (.csv files) from the MLML seawater intake or weather station. The data must be stored locally, and can be downloaded automatically with the download_station_data() function.

Inputs:
data_dir - Specifies the directory where the data files are located. All files with the format yyyy-mm.csv in this directory will be read.

Options:
    format: output format
        format = 'dict' (default): dictionary
        format = 'dataframe': pandas DataFrame
        format = 'dataset': xarray DataSet

Output: dictionary, pandas DataFrame or xarray DataSet with keys/variable names taken from column headers
    '''

    file_list = glob(data_dir+'*.csv')

    # get list of variable names from header of first file
    f = open(file_list[0],'r')
    header = f.readline()
    f.close()
    header = header.strip('\r\n')
    varnames = header.split(',')

    #initialize dictionary with key and empty list for each variable
    d = dict()
    for ii,var in enumerate(varnames[0:]):
        d[varnames[ii]] = []

    # specify which columns contain numeric data
    floatcols = range(2,len(varnames))
    allcols = range(0,len(varnames))
    strcols = list(set(allcols)-set(floatcols))

    for file_name in file_list:
        print('reading ' + file_name)

        # get numeric data, with missing values as NaN
        datamasked = np.genfromtxt(file_name,
                             skip_header=1,
                             delimiter=',',
                             missing_values='-99999',
                             usemask=True)
        data = datamasked.filled(np.nan)

        # get string data
        datastr = np.genfromtxt(file_name,
                         skip_header=1,
                         delimiter=',',
                         usecols=tuple(strcols),
                         dtype=str)

        # append data variables
        if data.size != 0:
            for col in floatcols:
                vname = varnames[col]
                d[vname] = np.append(d[vname],data[:,col])
            for si,col in enumerate(strcols):
                vname = varnames[col]
                d[vname] = np.append(d[vname],datastr[:,si])

    # create date variables
    # put in a numpy array for easy indexing
    # new variable for datetime
    dtime = np.array(util.list2date(d['utc_time'],'%Y-%m-%dT%H:%M:%SZ'))

    # remove duplicate times
    ii = np.where(np.diff(dtime) > timedelta(0.))[0]
    dtime = dtime[ii]
    for var in varnames:
        d[var] = d[var][ii]

    # Try loading in pandas or xarray format if specified, default to dictionary format
    if format == 'dataset':
        if 'xarray' not in sys.modules:
            format = 'dataframe'
            print("Warning: xarray not installed, loading MLML data in pandas dataframe format instead")
    if format == 'dataframe':
        if 'pandas' not in sys.modules:
            format = 'dict'
            print("Warning: pandas not installed, loading MLML data in dictionary format instead")

    if format is 'dataframe':
        # turn dictionary into pandas dataframe
        d = pd.DataFrame(d,index=dtime)
        d.index.name = 'time'
    elif format is 'dataset':
        # turn dictionary in xarray dataset, using dataframe as intermediate format
        d = pd.DataFrame(d,index=dtime)
        d.index.name = 'time'
        d = xr.Dataset(d)
        d.attrs['history'] = 'dataset created using physoce.obs.mlml.read_csv_data: ' + datetime.now().strftime("%Y-%m-%d %H:%M:%S") + ', '
    else:
        # default format: dictionary containing numpy arrays
        d['dtime'] = []
        d['dtime'] = dtime

    return d

def _add_metadata_xarray(d,station,readme_file):
    """
Add metadata to xarray dataset. Currently this adds lat and lon coordinates and puts the contents of the readme in an attribute. For the weather data, the anemometer height is also added as a coordinate.
    """

    if station is 'seawater':
        d.coords['lon'] = -121.7915
        d.coords['lat'] = 36.8025
    elif station is 'weather':
        d.coords['lon'] = -121.78842
        d.coords['lat'] = 36.80040
        d.coords['z'] = 3.3
        d.coords['z'].attrs['name'] = 'anemometer height'
        d.coords['z'].attrs['units'] = 'meters'

    with open(readme_file) as f:
        contents = f.read()
        d.attrs['readme'] = contents

    d.attrs['history'] = d.attrs['history'] + 'attributes added to dataset using physoce.obs.mlml._add_metadata_xarray: ' + datetime.now().strftime("%Y-%m-%d %H:%M:%S") + ', '


text:
import cv2
import numpy as np
import torch
import torch.nn as nn
import torch.utils.data
import torchvision.models as models
from PIL import Image
from torchvision import transforms

###################
# DETECT THE FACE #
###################

def face_detector(path):
    cascade = cv2.CascadeClassifier('./haarcascade_frontalface_alt2.xml')

    pil_image = Image.open(path).convert('RGB')
    open_cv_image = np.array(pil_image)
    open_cv_image = open_cv_image[:, :, ::-1].copy()

    grayscale = cv2.cvtColor(open_cv_image, cv2.COLOR_BGR2GRAY)
    detected_faces = cascade.detectMultiScale(grayscale)

    return len(detected_faces) > 0

##################################
# DETECT IF IT IS A CAT OR A DOG #
##################################

model_resnet18 = torch.hub.load('pytorch/vision', 'resnet18', pretrained=True)
model_resnet34 = torch.hub.load('pytorch/vision', 'resnet34', pretrained=True)

for name, param in model_resnet18.named_parameters():
    if ("bn" not in name):
        param.requires_grad = False

for name, param in model_resnet34.named_parameters():
    if ("bn" not in name):
        param.requires_grad = False

num_classes = 2

model_resnet18.fc = nn.Sequential(nn.Linear(model_resnet18.fc.in_features,512),
                                  nn.ReLU(),
                                  nn.Dropout(),
                                  nn.Linear(512, num_classes))

model_resnet34.fc = nn.Sequential(nn.Linear(model_resnet34.fc.in_features,512),
                                  nn.ReLU(),
                                  nn.Dropout(),
                                  nn.Linear(512, num_classes))

batch_size=32
img_dimensions = 224

# Normalize to the ImageNet mean and standard deviation
# Could calculate it for the cats/dogs data set, but the ImageNet
# values give acceptable results here.
img_transforms = transforms.Compose([
    transforms.Resize((img_dimensions, img_dimensions)),
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225] )
    ])

img_test_transforms = transforms.Compose([
    transforms.Resize((img_dimensions,img_dimensions)),
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225] )
    ])

if torch.cuda.is_available():
    device = torch.device("cuda")
else:
    device = torch.device("cpu")

resnet18 = torch.hub.load('pytorch/vision', 'resnet18')
resnet18.fc = nn.Sequential(nn.Linear(resnet18.fc.in_features,512),nn.ReLU(), nn.Dropout(), nn.Linear(512, num_classes))
resnet18.load_state_dict(torch.load('./model_resnet18.pth'))
resnet18.eval()

resnet34 = torch.hub.load('pytorch/vision', 'resnet34')
resnet34.fc = nn.Sequential(nn.Linear(resnet34.fc.in_features,512),nn.ReLU(), nn.Dropout(), nn.Linear(512, num_classes))
resnet34.load_state_dict(torch.load('./model_resnet34.pth'))
resnet34.eval()

models_ensemble = [resnet18.to(device), resnet34.to(device)]

def dog_or_cat_predict(img_path):
    image = Image.open(img_path)

    # These are the transforms according to the documentation in pytorch.models (https://pytorch.org/hub/pytorch_vision_vgg/)
    transform = transforms.Compose([transforms.Resize(img_dimensions),
                                    transforms.CenterCrop(img_dimensions),
                                    transforms.ToTensor(),
                                    transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                                         std=[0.229, 0.224, 0.225])])

    # Apply transforms and add new dimension (to input in the network)
    image_tensor = transform(image).unsqueeze(0)

    # move the input and model to GPU
    image_tensor = image_tensor.to(device)

    with torch.no_grad():
        predictions = [i(image_tensor).data for i in models_ensemble]
        avg_predictions = torch.mean(torch.stack(predictions), dim=0)
        _, predicted = torch.max(avg_predictions, 1)

        return predicted.item() == True

class_names = ['Affenpinscher', 'Afghan hound', 'Airedale terrier', 'Akita', 'Alaskan malamute', 'American eskimo dog',
               'American foxhound', 'American staffordshire terrier', 'American water spaniel',
               'Anatolian shepherd dog', 'Australian cattle dog', 'Australian shepherd', 'Australian terrier',
               'Basenji', 'Basset hound', 'Beagle', 'Bearded collie', 'Beauceron', 'Bedlington terrier',
               'Belgian malinois', 'Belgian sheepdog', 'Belgian tervuren', 'Bernese mountain dog', 'Bichon frise',
               'Black and tan coonhound', 'Black russian terrier', 'Bloodhound', 'Bluetick coonhound', 'Border collie',
               'Border terrier', 'Borzoi', 'Boston terrier', 'Bouvier des flandres', 'Boxer', 'Boykin spaniel',
               'Briard', 'Brittany', 'Brussels griffon', 'Bull terrier', 'Bulldog', 'Bullmastiff', 'Cairn terrier',
               'Canaan dog', 'Cane corso', 'Cardigan welsh corgi', 'Cavalier king charles spaniel',
               'Chesapeake bay retriever', 'Chihuahua', 'Chinese crested', 'Chinese shar-pei', 'Chow chow',
               'Clumber spaniel', 'Cocker spaniel', 'Collie', 'Curly-coated retriever', 'Dachshund', 'Dalmatian',
               'Dandie dinmont terrier', 'Doberman pinscher', 'Dogue de bordeaux', 'English cocker spaniel',
               'English setter', 'English springer spaniel', 'English toy spaniel', 'Entlebucher mountain dog',
               'Field spaniel', 'Finnish spitz', 'Flat-coated retriever', 'French bulldog', 'German pinscher',
               'German shepherd dog', 'German shorthaired pointer', 'German wirehaired pointer', 'Giant schnauzer',
               'Glen of imaal terrier', 'Golden retriever', 'Gordon setter', 'Great dane', 'Great pyrenees',
               'Greater swiss mountain dog', 'Greyhound', 'Havanese', 'Ibizan hound', 'Icelandic sheepdog',
               'Irish red and white setter', 'Irish setter', 'Irish terrier', 'Irish water spaniel', 'Irish wolfhound',
               'Italian greyhound', 'Japanese chin', 'Keeshond', 'Kerry blue terrier', 'Komondor', 'Kuvasz',
               'Labrador retriever', 'Lakeland terrier', 'Leonberger', 'Lhasa apso', 'Lowchen', 'Maltese',
               'Manchester terrier', 'Mastiff', 'Miniature schnauzer', 'Neapolitan mastiff', 'Newfoundland',
               'Norfolk terrier', 'Norwegian buhund', 'Norwegian elkhound', 'Norwegian lundehund', 'Norwich terrier',
               'Nova scotia duck tolling retriever', 'Old english sheepdog', 'Otterhound', 'Papillon',
               'Parson russell terrier', 'Pekingese', 'Pembroke welsh corgi', 'Petit basset griffon vendeen',
               'Pharaoh hound', 'Plott', 'Pointer', 'Pomeranian', 'Poodle', 'Portuguese water dog', 'Saint bernard',
               'Silky terrier', 'Smooth fox terrier', 'Tibetan mastiff', 'Welsh springer spaniel',
               'Wirehaired pointing griffon', 'Xoloitzcuintli', 'Yorkshire terrier']

###############################
# DETECT BREED IF IT IS A DOG #
###############################

def predict_breed(model, img_path):
    image = Image.open(img_path).convert('RGB')

    mean = [0.485, 0.456, 0.406]
    std = [0.229, 0.224, 0.225]

    # Declaring transforms and normalizing according to the documentation
    transform = transforms.Compose([transforms.Resize(256),
                                    transforms.CenterCrop(256),
                                    transforms.ToTensor(),
                                    transforms.Normalize(mean=mean, std=std)])

    # discard the transparent, alpha channel and adding dimension (just as in the VGG prediction)
    image = transform(image)[:3, :, :].unsqueeze(0)

    model = model.to("cpu")  # changed to CPU because of it is expecting a floattensor, not a cuda.floattensor
    model.eval()
    idx = torch.argmax(model(image))  # the index with the highest prediction

    return class_names[idx]

model_transfer = models.resnet18(pretrained=True)

# Freeze parameters so we don't backprop through them
for param in model_transfer.parameters():
    param.requires_grad = False

# Define new classifier, two fully connected layers with ReLU activation function and dropout
classifier = nn.Sequential(nn.Linear(512, 256),
                           nn.ReLU(),
                           nn.Dropout(0.3),
                           nn.Linear(256, 133),
                           nn.LogSoftmax(dim=1))

model_transfer.fc = classifier

# check if CUDA is available
use_cuda = torch.cuda.is_available()

if use_cuda:
    model_transfer = model_transfer.cuda()

model_transfer.load_state_dict(torch.load('./saved_model.pt', map_location=torch.device('cpu')))

########################################
# ADDITIONAL SUPPORT FOR DOGS AND CATS #
########################################

VGG16 = models.vgg16(pretrained=True)
use_cuda = torch.cuda.is_available()
if use_cuda:
    VGG16 = VGG16.cuda()

def VGG16_predict(img_path):
    image = Image.open(img_path).convert('RGB')

    # These are the transforms according to the documentation in pytorch.models (
    # https://pytorch.org/hub/pytorch_vision_vgg/)
    transform = transforms.Compose([transforms.Resize(256),
                                    transforms.CenterCrop(224),
                                    transforms.ToTensor(),
                                    transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                                         std=[0.229, 0.224, 0.225])])

    # Apply transforms and add new dimension (to input in the network)
    image_tensor = transform(image).unsqueeze(0)

    # move the input and model to GPU
    if use_cuda:
        image_tensor = image_tensor.to('cuda')

    # We turn eval() mode on before prediction and resume training afterwards
    VGG16.eval()

    # Get prediction
    with torch.no_grad():
        output = VGG16(image_tensor)
        prediction = torch.argmax(output).item()  # Returns the index of the maximum value

    VGG16.train()

    return prediction

def dog_detector(img_path):
    prediction_index = VGG16_predict(img_path)
    return True if (269 > prediction_index > 150) else False

def cat_detector(img_path):
    prediction_index = VGG16_predict(img_path)
    return True if (288 > prediction_index > 280) else False

##################
# FINAL FUNCTION #
##################

def detect_image(img_path):
    ret_str = ""
    if face_detector(img_path) is True:
        ret_str += "There is a human in this photo.\n"

    if dog_or_cat_predict(img_path) is True:
        ret_str += "There is a high probability this has a dog...\n"
        if dog_detector(img_path) is True:
            pred = predict_breed(model_transfer, img_path)
            ret_str += "its breed being {0}.\n".format(pred)

    elif cat_detector(img_path) is True:
        ret_str += "I'm quite certain the picture has a cat!\n"

    else:
        ret_str += "There is not a human nor there is a dog, so perhaps a cat...?\n"

    return ret_str

max_stars_repo_name: LuchoTurtle/dog-breed-classifier

max_issues_repo_name: LuchoTurtle/dog-breed-classifier

max_forks_repo_name: LuchoTurtle/dog-breed-classifier


text:
import os
from os.path import exists, join
from pathlib import Path
import logging
import json
from random import shuffle

import numpy as np

from .base_dataset import BaseDataset
from ..utils import make_dir, DATASET

logging.basicConfig(
    level=logging.INFO,
    format='%(levelname)s - %(asctime)s - %(module)s - %(message)s',
)
log = logging.getLogger(__name__)

class ShapeNet(BaseDataset):
    """This class is used to create a dataset based on the ShapeNet dataset, and
    used in object detection, visualizer, training, or testing.

    The ShapeNet dataset includes a large set of 3D shapes.
    """

    def __init__(self,
                 dataset_path,
                 name="ShapeNet",
                 class_weights=[
                     2690, 76, 55, 1824, 3746, 69, 787, 392, 1546, 445, 202,
                     184, 275, 66, 152, 5266
                 ],
                 ignored_label_inds=[],
                 test_result_folder='./test',
                 task="classification",
                 **kwargs):
        """Initialize the function by passing the dataset and other details.

        Args:
            dataset_path: The path to the dataset to use.
            name: The name of the dataset (ShapeNet in this case).
            class_weights: The class weights to use in the dataset.
            ignored_label_inds: A list of labels that should be ignored in the dataset.
            test_result_folder: The folder where the test results should be stored.
            task: The task that identifies the purpose. The valid values are classification and segmentation.

        Returns:
            class: The corresponding class.
        """
        super().__init__(dataset_path=dataset_path,
                         name=name,
                         cache_dir='./logs/cache',
                         use_cache=False,
                         task=task,
                         class_weights=class_weights,
                         ignored_label_inds=ignored_label_inds,
                         test_result_folder=test_result_folder,
                         **kwargs)

        assert task in ['classification',
                        'segmentation'], f"Invalid task {task}"

        self.label_to_names = self.get_label_to_names()
        self.num_classes = len(self.label_to_names)
        self.dataset_path = join(
            dataset_path, 'shapenetcore_partanno_segmentation_benchmark_v0')
        self.task = task

        self.cat = {}
        self.catfile = os.path.join(self.dataset_path,
                                    'synsetoffset2category.txt')
        with open(self.catfile, 'r') as f:
            for idx, line in enumerate(f):
                ls = line.strip().split()
                self.cat[idx] = ls[1]

        self.meta = {}
        for item in self.cat:
            self.meta[item] = []
            dir_point = os.path.join(self.dataset_path, self.cat[item],
                                     'points')
            dir_seg = os.path.join(self.dataset_path, self.cat[item],
                                   'points_label')
            fns = sorted(os.listdir(dir_point))
            for fn in fns:
                token = (os.path.splitext(os.path.basename(fn))[0])
                self.meta[item].append(
                    (join(dir_point,
                          token + '.pts'), join(dir_seg, token + '.seg')))

        splits = []
        splits_path = join(self.dataset_path, 'train_test_split')
        for split in [
                'shuffled_train_file_list.json', 'shuffled_test_file_list.json',
                'shuffled_val_file_list.json'
        ]:
            with open(join(splits_path, split)) as source:
                json_source = source.read()
                splits.append(
                    [i.split('/')[-1] for i in json.loads(json_source)])
        train_split, test_split, val_split = splits

        self.all_files = []
        self.train_files = []
        self.val_files = []
        self.test_files = []
        for item in self.cat:
            for fn in self.meta[item]:
                self.all_files.append((item, fn[0], fn[1]))
                file = fn[0].split('/')[-1].split('.')[0]
                if file in train_split:
                    self.train_files.append((item, fn[0], fn[1]))
                elif file in test_split:
                    self.test_files.append((item, fn[0], fn[1]))
                else:
                    self.val_files.append((item, fn[0], fn[1]))
        shuffle(self.train_files)
        shuffle(self.test_files)
        shuffle(self.val_files)

    @staticmethod
    def get_label_to_names(task="classification"):
        """Returns a label to names dictionary object depending on the task. The
        valid values for task for classification and segmentation.

        Returns:
            A dict where keys are label numbers and values are the corresponding
            names.
        """
        if task == "classification":
            label_to_names = {
                0: 'Airplane',
                1: 'Bag',
                2: 'Cap',
                3: 'Car',
                4: 'Chair',
                5: 'Earphone',
                6: 'Guitar',
                7: 'Knife',
                8: 'Lamp',
                9: 'Laptop',
                10: 'Motorbike',
                11: 'Mug',
                12: 'Pistol',
                13: 'Rocket',
                14: 'Skateboard',
                15: 'Table'
            }
        elif task == "segmentation":
            label_to_names = {}
            for i in range(50):
                label_to_names[i] = f"Part{i}"
        else:
            raise ValueError(f"Invalid task {task}")
        return label_to_names

    def get_split(self, split):
        """Returns a dataset split.

        Args:
            split: A string identifying the dataset split that is usually one of
            'training', 'test', 'validation', or 'all'.

        Returns:
            A dataset split object providing the requested subset of the data.
        """
        return ShapeNetSplit(self, split=split, task=self.task)

    def get_split_list(self, split):
        """Returns the list of data splits available.

        Args:
            split: A string identifying the dataset split that is usually one of
            'training', 'test', 'validation', or 'all'.

        Returns:
            A dataset split object providing the requested subset of the data.

        Raises:
            ValueError: Indicates that the split name passed is incorrect. The split name should be one of
            'training', 'test', 'validation', or 'all'.
        """
        if split in ['test', 'testing']:
            files = self.test_files
        elif split in ['train', 'training']:
            files = self.train_files
        elif split in ['val', 'validation']:
            files = self.val_files
        elif split in ['all']:
            files = self.val_files + self.train_files + self.test_files
        else:
            raise ValueError(f"Invalid split {split}")
        return files

    def is_tested(self, attr):
        """Checks if a datum in the dataset has been tested.

        Args:
            attr: The attribute that needs to be checked.

        Returns:
            If the datum attribute is tested, then return the path where the
                attribute is stored; else, returns false.
        """
        cfg = self.cfg
        name = attr['name']
        path = cfg.test_result_folder
        store_path = join(path, self.name, name + '.labels')
        if exists(store_path):
            print("{} already exists.".format(store_path))
            return True
        else:
            return False

    def save_test_result(self, results, attr):
        """Saves the output of a model.

        Args:
            results: The output of a model for the datum associated with the attribute passed.
            attr: The attributes that correspond to the outputs passed in results.
        """
        cfg = self.cfg
        name = attr['name']
        path = cfg.test_result_folder
        make_dir(path)

        pred = results['predict_labels'] + 1
        store_path = join(path, self.name, name + '.labels')
        make_dir(Path(store_path).parent)
        np.savetxt(store_path, pred.astype(np.int32), fmt='%d')

        log.info("Saved {} in {}.".format(name, store_path))

class ShapeNetSplit:
    """The class gets data and attributes based on the split and
    classification.
    """

    def __init__(self, dataset, split='training', task='classification'):
        assert task in ['classification',
                        'segmentation'], f"Invalid task {task}"

        self.cfg = dataset.cfg
        path_list = dataset.get_split_list(split)

        self.path_list = path_list
        self.split = split
        self.dataset = dataset
        self.task = task

    def __len__(self):
        return len(self.path_list)

    def get_data(self, idx):
        path = self.path_list[idx]
        points = np.loadtxt(path[1], dtype=np.float32)
        label = np.loadtxt(
            path[2],
            dtype=np.int64) if self.task == 'segmentation' else np.array(
                [np.int64(path[0])])
        return {'point': points, 'feat': None, 'label': label}

    def get_attr(self, idx):
        name = self.path_list[idx][1].split('/')[-1].split('.')[0]
        return {
            'name': name,
            'path': str(Path(self.path_list[idx][1])),
            'split': self.split
        }

DATASET._register_module(ShapeNet)


text:
import tempfile
from os.path import join
import subprocess
import re
import sys
import os
import numpy as np

BLEU_PATH = join("scripts", "multi-bleu.perl")

BASE_REF_FNAME = "ref"
HYP_FNAME = "hyp"

def call_bleu(base_ref_fname, hyp_fname):
    command = "perl %s %s < %s" % (BLEU_PATH, base_ref_fname, hyp_fname)
    result = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)\
        .stdout.read().decode("utf-8")

    match = re.match("BLEU = ([\d.]+),.*BP=([\d.]+),.*\)", result)
    if match is not None:
        # print(match.group(0))
        bleu = float(match.group(1))
        brevity_penalty = float(match.group(2))
        unpenalized = 0
        if brevity_penalty != 0:
            unpenalized = bleu / brevity_penalty
        return bleu, unpenalized

    else:
        sys.stderr.write(
            "warning: BLEU score not found in output file, returning 0")
        return 0, 0

def read_file(fname):
    with open(fname) as f:
        return [line.split() for line in f]

def multi_bleu(multiple_references, hypotheses):
    dir = tempfile.mkdtemp()
    num_refs = len(multiple_references[0])
    assert(all(len(l) == num_refs for l in multiple_references))

    base_ref_fname = join(dir, BASE_REF_FNAME)
    for i in range(num_refs):
        ref_fname = base_ref_fname + str(i)
        with open(ref_fname, 'w') as f:
            for refs in multiple_references:
                f.write("%s\n" % ' '.join(refs[i]))

    hyp_fname = join(dir, HYP_FNAME)
    with open(hyp_fname, 'w') as f:
        for hyp in hypotheses:
            f.write("%s\n" % ' '.join(hyp))

    bleu, unpenalized_bleu = call_bleu(base_ref_fname, hyp_fname)

    # clean up
    for i in range(num_refs):
        ref_fname = base_ref_fname + str(i)
        os.remove(ref_fname)

    os.remove(hyp_fname)
    os.rmdir(dir)

    return bleu, unpenalized_bleu

def single_bleu( references, hypotheses):
    return multi_bleu([[ref] for ref in references], hypotheses)

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("ref_fname")
    parser.add_argument("hyp_fname")
    parser.add_argument("--sentence_level", action='store_true')
    parser.add_argument("--nltk", action='store_true')
    args = parser.parse_args()

    refs = read_file(args.ref_fname)
    hyps = read_file(args.hyp_fname)

    # f_bleu = call_bleu(args.ref_fname, args.hyp_fname)
    # l_bleu = multi_bleu([[r] for r in refs], hyps)
    # c_bleu = single_bleu(refs, hyps)
    # assert(f_bleu == l_bleu == c_bleu)

    if args.sentence_level:
        scores = []
        for (ref, hyp) in zip(refs, hyps):
            if args.nltk:
                import nltk
                scores.append(
                    nltk.translate.bleu_score.sentence_bleu([ref], hyp))
            else:
                scores.append(single_bleu([ref], [hyp])[0])
        result = np.mean(scores)
    else:
        if args.nltk:
            import nltk
            result = nltk.translate.bleu_score.corpus_bleu(
                [[r] for r in refs], hyps)
        result = single_bleu(refs, hyps)[0]

    print(result)


text:
import streamlit as st
import numpy as np
import pandas as pd
import json
import pickle
import reference_book as rb

def main():
    model = load_data()
    st.title('Questionario - Open Sex Role Inventory')
    st.text(
        """
            Olá, bem vindo!

            Este questionario foi baseado em um estudo chamado Open Sex-Role Inventory, realizado originalmente por Sandra Bam e visava ser capaz de estudar e prever a
            sexualidade das pessoas utilizando uma serie de perguntas. Atualmente, este questionario é muito mais amplo(apesar de parecer dubio) e mede a sexualidade de um
            individuo de forma mais complexa.

            A ideia deste questionario que você está prestes a fazer não tem nada haver com o objetivo original do teste, mas, em utilizar os dados fornecidos pelos
            pesquisadores responsáveis para prever algumas outras informações da sua pessoa, sendo meramente um experimento envolvendo Estatística, Inteligencia Artificial
            e Programação.

            Os dados informados aqui são completamente anónimos e utilizados apenas para propositos de medição da qualidade do modelo de decisão utilizado no experimento.

            O questionario deve ser respondido da seguinte forma:
                Você será exposto a 44 perguntas que devem ser respondidas utilizando um slider de intensidade, onde o valor 1(mais baixo) representa total discordancia com a
                afirmação, o valor 3(Médio) representa neutralidade sobre a pergunta e o valor 5(Mais alto) representa total concordancia com a pergunta.

            Dessa forma, você deve responder o questionario da forma mais honesta possível. Boa sorte!

            1 - Discordo totalmente;
            2 - Discordo;
            3 - Neutro;
            4 - Concordo;
            5 - Concordo totalmente.
        """
    )

    st.sidebar.title("OSRI Questionario")
    confirm_btn = st.sidebar.button(
        label='Confirmar questionario'
    )

    # Question sliders
    Q1 = st.slider(
        label='Eu estudei como ganhar apostando',
        min_value=1,
        max_value=5,
        step=1)

    Q2 = st.slider(
        label='Eu penso em pintar meu cabelo',
        min_value=1,
        max_value=5,
        step=1)

    Q3 = st.slider(
        label='Eu gosto de arremessar facas, machados e outras coisas cortantes',
        min_value=1,
        max_value=5,
        step=1)

    Q4 = st.slider(
        label='Eu presenteio as pessoas com presentes feitos a mão',
        min_value=1,
        max_value=5,
        step=1)

    Q5 = st.slider(
        label='Eu tenho sonhos em que estou salvando alguém de um prédio em chamas',
        min_value=1,
        max_value=5,
        step=1)

    Q6 = st.slider(
        label='Eu fico envergonhado quando alguém lê algo que eu escrevi',
        min_value=1,
        max_value=5,
        step=1)

    Q7 = st.slider(
        label='Eu ja estive/estou muito interessadas em guerras históricas',
        min_value=1,
        max_value=5,
        step=1)

    Q8 = st.slider(
        label='Eu sei os aniversários dos meus amigos',
        min_value=1,
        max_value=5,
        step=1)

    Q9 = st.slider(
        label='Eu gosto de armas de fogo',
        min_value=1,
        max_value=5,
        step=1)

    Q10 = st.slider(
        label='Eu fico mais feliz quando estou na minha cama',
        min_value=1,
        max_value=5,
        step=1)

    Q11 = st.slider(
        label='Eu não trabalhei/estudei muito no ensino médio',
        min_value=1,
        max_value=5,
        step=1)

    Q12 = st.slider(
        label='Eu uso loção para minhas mãos',
        min_value=1,
        max_value=5,
        step=1)

    Q13 = st.slider(
        label='Eu prefiro ter aulas de matemática do que aulas de artes manuais',
        min_value=1,
        max_value=5,
        step=1)

    Q14 = st.slider(
        label='Eu danço quando estou sozinho',
        min_value=1,
        max_value=5,
        step=1)

    Q15 = st.slider(
        label='Eu penso/pensei que seria muito excitante se tornar um fora da lei',
        min_value=1,
        max_value=5,
        step=1)

    Q16 = st.slider(
        label='Quando eu era criança, eu costumava brincar de fingir que estava em uma banda com meus amigos',
        min_value=1,
        max_value=5,
        step=1)

    Q17 = st.slider(
        label='Eu considero/considerei entrar para as forças armadas',
        min_value=1,
        max_value=5,
        step=1)

    Q18 = st.slider(
        label='Eu fico tonto quando me levanto bruscamente',
        min_value=1,
        max_value=5,
        step=1)

    Q19 = st.slider(
        label='Eu não considero normal ficar com raiva/triste ao ouvir sobre a morte de pessoas que você não conhece',
        min_value=1,
        max_value=5,
        step=1)

    Q20 = st.slider(
        label='Algumas vezes eu choro quando fico com muita raiva',
        min_value=1,
        max_value=5,
        step=1)

    Q21 = st.slider(
        label='Eu não lembro de datas de aniversários',
        min_value=1,
        max_value=5,
        step=1)

    Q22 = st.slider(
        label='Eu guardo as cartas que eu recebo',
        min_value=1,
        max_value=5,
        step=1)

    Q23 = st.slider(
        label='Eu brinco de xingar meus amigos e não me importo que façam o mesmo',
        min_value=1,
        max_value=5,
        step=1)

    Q24 = st.slider(
        label='Sou contra experimentos médicos em animais',
        min_value=1,
        max_value=5,
        step=1)

    Q25 = st.slider(
        label='Eu posso fazer uma quantidade incrível de flexões',
        min_value=1,
        max_value=5,
        step=1)

    Q26 = st.slider(
        label='Eu pulo quando fico muito animado',
        min_value=1,
        max_value=5,
        step=1)

    Q27 = st.slider(
        label='Eu penso que um desastre natural poderia ser divertido',
        min_value=1,
        max_value=5,
        step=1)

    Q28 = st.slider(
        label='Eu ando com um cobertor pela casa',
        min_value=1,
        max_value=5,
        step=1)

    Q29 = st.slider(
        label='Eu costumava/costumo queimar coisas com a luz do sol e uma lupa',
        min_value=1,
        max_value=5,
        step=1)

    Q30 = st.slider(
        label='Eu acho o horóscopo divertido',
        min_value=1,
        max_value=5,
        step=1)

    Q31 = st.slider(
        label='Eu não levo muita bagagem quando viajo',
        min_value=1,
        max_value=5,
        step=1)

    Q32 = st.slider(
        label='Eu ja pensei/penso em me tornar vegetariano',
        min_value=1,
        max_value=5,
        step=1)

    Q33 = st.slider(
        label='Eu odeio sair as compras',
        min_value=1,
        max_value=5,
        step=1)

    Q34 = st.slider(
        label='Eu tenho/tive um diario pessoal que guardo até hoje',
        min_value=1,
        max_value=5,
        step=1)

    Q35 = st.slider(
        label='Eu desmontei/desmonto maquinas apenas para ver como elas funcionam',
        min_value=1,
        max_value=5,
        step=1)

    Q36 = st.slider(
        label='Eu tenho muitas fotos das coisas que eu fiz/faço',
        min_value=1,
        max_value=5,
        step=1)

    Q37 = st.slider(
        label='Eu ja joguei/jogo muitos video games',
        min_value=1,
        max_value=5,
        step=1)

    Q38 = st.slider(
        label='De vez em quando deixo boas mensagens para as pessoas',
        min_value=1,
        max_value=5,
        step=1)

    Q39 = st.slider(
        label='Eu ja botei fogo em vários tipos de combustíveis apenas por diversão',
        min_value=1,
        max_value=5,
        step=1)

    Q40 = st.slider(
        label='Eu realmente gosto de dançar',
        min_value=1,
        max_value=5,
        step=1)

    Q41 = st.slider(
        label='Em uma escada, subo dois degraus de cada vez',
        min_value=1,
        max_value=5,
        step=1)

    Q42 = st.slider(
        label='Eu cozinho doces e coisas gostosas para mim mesmo as vezes',
        min_value=1,
        max_value=5,
        step=1)

    Q43 = st.slider(
        label='Eu penso que um desastre natural poderia ser divertido',
        min_value=1,
        max_value=5,
        step=1)

    Q44 = st.slider(
        label='Eu decoro meus pertences(Ex: adesivos no notebook)',
        min_value=1,
        max_value=5,
        step=1)

    # Features array
    features = np.array([
        Q1,Q2,Q3,Q4,Q5,Q6,Q7,Q8,Q9,Q10,Q11,Q12,Q13,Q14,
        Q15,Q16,Q17,Q18,Q19,Q20,Q21,Q22,Q23,Q24,Q25,Q26,
        Q27,Q28,Q29,Q30,Q31,Q32,Q33,Q34,Q35,Q36,Q37,Q38,
        Q39,Q40,Q41,Q42,Q43,Q44
    ])

    if confirm_btn:
        ypred = model.predict(features.reshape([1, -1]))

        age = ypred[0]
        education = rb.education[ypred[1]]
        gender = rb.gender[ypred[2]]
        orientation = rb.orientation[ypred[3]]
        race = rb.race[ypred[4]]
        religion = rb.religion[ypred[5]]
        hand = rb.hand[ypred[6]]
        print(ypred)

@st.cache
def load_data():
    with open("files/OSRI_Decision_Tree_model.hdf5", 'rb') as model_file:
        return pickle.load(model_file)

if __name__ == "__main__":
    main()

if __name__ != "__main__":
    main()


text:
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for training utility functions."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from absl.testing import parameterized
import numpy as np

from tensorflow.python.data.ops import dataset_ops
from tensorflow.python.data.ops import readers
from tensorflow.python.eager import context
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import tensor_util
from tensorflow.python.keras import backend
from tensorflow.python.keras import keras_parameterized
from tensorflow.python.keras import testing_utils
from tensorflow.python.keras.engine import training_utils
from tensorflow.python.keras.utils import tf_utils
from tensorflow.python.platform import test
from tensorflow.python.platform import tf_logging as logging

class ModelInputsTest(test.TestCase):

  def test_single_thing(self):
    a = np.ones(10)
    model_inputs = training_utils.ModelInputs(a)
    self.assertEquals(['input_1'], model_inputs.get_input_names())
    vals = model_inputs.get_input_values()
    self.assertAllEqual(np.ones(10), vals)
    self.assertFalse(tensor_util.is_tensor(vals))
    vals = model_inputs.get_symbolic_inputs()
    self.assertTrue(tensor_util.is_tensor(vals))
    vals = model_inputs.get_symbolic_inputs(return_single_as_list=True)
    self.assertEquals(1, len(vals))
    self.assertTrue(tensor_util.is_tensor(vals[0]))
    self.assertEqual(backend.floatx(), vals[0].dtype)

  def test_single_thing_eager(self):
    with context.eager_mode():
      a = np.ones(10, dtype=np.int32)
      model_inputs = training_utils.ModelInputs(a)
      self.assertEquals(['input_1'], model_inputs.get_input_names())
      vals = model_inputs.get_input_values()
      self.assertAllEqual(np.ones(10), vals)
      self.assertTrue(tensor_util.is_tensor(vals))
      vals = model_inputs.get_symbolic_inputs()
      self.assertTrue(isinstance(vals, base_layer.DeferredTensor))
      vals = model_inputs.get_symbolic_inputs(return_single_as_list=True)
      self.assertEqual(1, len(vals))
      self.assertTrue(tf_utils.is_symbolic_tensor(vals[0]))
      self.assertEqual(dtypes.int32, vals[0].dtype)

  def test_list(self):
    a = [np.ones(10), np.ones(20)]
    model_inputs = training_utils.ModelInputs(a)
    self.assertEquals(['input_1', 'input_2'], model_inputs.get_input_names())
    vals = model_inputs.get_input_values()
    self.assertEqual(2, len(vals))
    self.assertAllEqual(np.ones(10), vals[0])
    self.assertAllEqual(np.ones(20), vals[1])
    self.assertFalse(tensor_util.is_tensor(vals[0]))
    self.assertFalse(tensor_util.is_tensor(vals[1]))
    vals = model_inputs.get_symbolic_inputs()
    self.assertTrue(tensor_util.is_tensor(vals[0]))
    self.assertTrue(tensor_util.is_tensor(vals[1]))

  def test_list_eager(self):
    with context.eager_mode():
      a = [np.ones(10), np.ones(20)]
      model_inputs = training_utils.ModelInputs(a)
      self.assertEquals(['input_1', 'input_2'], model_inputs.get_input_names())
      vals = model_inputs.get_input_values()
      self.assertEqual(2, len(vals))
      self.assertAllEqual(np.ones(10), vals[0])
      self.assertAllEqual(np.ones(20), vals[1])
      self.assertTrue(tensor_util.is_tensor(vals[0]))
      self.assertTrue(tensor_util.is_tensor(vals[1]))
      vals = model_inputs.get_symbolic_inputs()
      self.assertTrue(isinstance(vals[0], base_layer.DeferredTensor))
      self.assertTrue(isinstance(vals[1], base_layer.DeferredTensor))

  def test_dict(self):
    a = {'b': np.ones(10), 'a': np.ones(20)}
    model_inputs = training_utils.ModelInputs(a)
    self.assertEquals(['a', 'b'], model_inputs.get_input_names())
    vals = model_inputs.get_input_values()
    self.assertAllEqual(np.ones(20), vals['a'])
    self.assertAllEqual(np.ones(10), vals['b'])
    self.assertFalse(tensor_util.is_tensor(vals['a']))
    self.assertFalse(tensor_util.is_tensor(vals['b']))
    vals = model_inputs.get_symbolic_inputs()
    self.assertTrue(tensor_util.is_tensor(vals['a']))
    self.assertTrue(tensor_util.is_tensor(vals['b']))

  def test_dict_eager(self):
    with context.eager_mode():
      a = {'b': np.ones(10), 'a': np.ones(20)}
      model_inputs = training_utils.ModelInputs(a)
      self.assertEquals(['a', 'b'], model_inputs.get_input_names())
      vals = model_inputs.get_input_values()
      self.assertAllEqual(np.ones(20), vals['a'])
      self.assertAllEqual(np.ones(10), vals['b'])
      self.assertTrue(tensor_util.is_tensor(vals['a']))
      self.assertTrue(tensor_util.is_tensor(vals['b']))
      vals = model_inputs.get_symbolic_inputs()
      self.assertTrue(tf_utils.is_symbolic_tensor(vals['a']))
      self.assertTrue(tf_utils.is_symbolic_tensor(vals['b']))

class DatasetUtilsTest(test.TestCase, parameterized.TestCase):

  @parameterized.named_parameters(
      # pylint: disable=g-long-lambda
      ('Batch', lambda: dataset_ops.Dataset.range(5).batch(2), ValueError),
      ('Cache', lambda: dataset_ops.Dataset.range(5).cache()),
      ('Concatenate', lambda: dataset_ops.Dataset.range(5).concatenate(
          dataset_ops.Dataset.range(5))),
      ('FlatMap', lambda: dataset_ops.Dataset.range(5).flat_map(
          lambda _: dataset_ops.Dataset.from_tensors(0)), ValueError),
      ('Filter', lambda: dataset_ops.Dataset.range(5).filter(lambda _: True)),
      ('FixedLengthRecordDatasetV2',
       lambda: readers.FixedLengthRecordDatasetV2([], 42)),
      ('FromTensors', lambda: dataset_ops.Dataset.from_tensors(0)),
      ('FromTensorSlices',
       lambda: dataset_ops.Dataset.from_tensor_slices([0, 0, 0])),
      ('Interleave', lambda: dataset_ops.Dataset.range(5).interleave(
          lambda _: dataset_ops.Dataset.from_tensors(0), cycle_length=1),
       ValueError),
      ('ParallelInterleave', lambda: dataset_ops.Dataset.range(5).interleave(
          lambda _: dataset_ops.Dataset.from_tensors(0),
          cycle_length=1,
          num_parallel_calls=1), ValueError),
      ('Map', lambda: dataset_ops.Dataset.range(5).map(lambda x: x)),
      ('Options',
       lambda: dataset_ops.Dataset.range(5).with_options(dataset_ops.Options())
      ),
      ('PaddedBatch', lambda: dataset_ops.Dataset.range(5).padded_batch(2, []),
       ValueError),
      ('ParallelMap', lambda: dataset_ops.Dataset.range(5).map(
          lambda x: x, num_parallel_calls=1)),
      ('Prefetch', lambda: dataset_ops.Dataset.range(5).prefetch(1)),
      ('Range', lambda: dataset_ops.Dataset.range(0)),
      ('Repeat', lambda: dataset_ops.Dataset.range(0).repeat(0)),
      ('Shuffle', lambda: dataset_ops.Dataset.range(5).shuffle(1)),
      ('Skip', lambda: dataset_ops.Dataset.range(5).skip(2)),
      ('Take', lambda: dataset_ops.Dataset.range(5).take(2)),
      ('TextLineDataset', lambda: readers.TextLineDatasetV2([])),
      ('TFRecordDataset', lambda: readers.TFRecordDatasetV2([])),
      ('Window', lambda: dataset_ops.Dataset.range(5).window(2), ValueError),
      ('Zip', lambda: dataset_ops.Dataset.zip(dataset_ops.Dataset.range(5))),
      # pylint: enable=g-long-lambda
  )
  def test_assert_not_batched(self, dataset_fn, expected_error=None):
    if expected_error is None:
      training_utils.assert_not_batched(dataset_fn())
    else:
      with self.assertRaises(expected_error):
        training_utils.assert_not_batched(dataset_fn())

  @parameterized.named_parameters(
      # pylint: disable=g-long-lambda
      ('Batch', lambda: dataset_ops.Dataset.range(5).batch(2)),
      ('Cache', lambda: dataset_ops.Dataset.range(5).cache()),
      ('Concatenate', lambda: dataset_ops.Dataset.range(5).concatenate(
          dataset_ops.Dataset.range(5))),
      ('FlatMap', lambda: dataset_ops.Dataset.range(5).flat_map(
          lambda _: dataset_ops.Dataset.from_tensors(0)), ValueError),
      ('Filter', lambda: dataset_ops.Dataset.range(5).filter(lambda _: True)),
      ('FixedLengthRecordDatasetV2',
       lambda: readers.FixedLengthRecordDatasetV2([], 42)),
      ('FromTensors', lambda: dataset_ops.Dataset.from_tensors(0)),
      ('FromTensorSlices',
       lambda: dataset_ops.Dataset.from_tensor_slices([0, 0, 0])),
      ('Interleave', lambda: dataset_ops.Dataset.range(5).interleave(
          lambda _: dataset_ops.Dataset.from_tensors(0), cycle_length=1),
       ValueError),
      ('Map', lambda: dataset_ops.Dataset.range(5).map(lambda x: x)),
      ('Options',
       lambda: dataset_ops.Dataset.range(5).with_options(dataset_ops.Options())
      ),
      ('PaddedBatch', lambda: dataset_ops.Dataset.range(5).padded_batch(2, [])),
      ('ParallelInterleave', lambda: dataset_ops.Dataset.range(5).interleave(
          lambda _: dataset_ops.Dataset.from_tensors(0),
          cycle_length=1,
          num_parallel_calls=1), ValueError),
      ('ParallelMap', lambda: dataset_ops.Dataset.range(5).map(
          lambda x: x, num_parallel_calls=1)),
      ('Prefetch', lambda: dataset_ops.Dataset.range(5).prefetch(1)),
      ('Range', lambda: dataset_ops.Dataset.range(0)),
      ('Repeat', lambda: dataset_ops.Dataset.range(0).repeat(0)),
      ('Shuffle', lambda: dataset_ops.Dataset.range(5).shuffle(1), ValueError),
      ('Skip', lambda: dataset_ops.Dataset.range(5).skip(2)),
      ('Take', lambda: dataset_ops.Dataset.range(5).take(2)),
      ('TextLineDataset', lambda: readers.TextLineDatasetV2([])),
      ('TFRecordDataset', lambda: readers.TFRecordDatasetV2([])),
      ('Window', lambda: dataset_ops.Dataset.range(5).window(2)),
      ('Zip', lambda: dataset_ops.Dataset.zip(dataset_ops.Dataset.range(5))),
      # pylint: enable=g-long-lambda
  )
  def test_assert_not_shuffled(self, dataset_fn, expected_error=None):
    if expected_error is None:
      training_utils.assert_not_shuffled(dataset_fn())
    else:
      with self.assertRaises(expected_error):
        training_utils.assert_not_shuffled(dataset_fn())

  def test_verify_dataset_shuffled(self):
    dataset = dataset_ops.Dataset.range(5)
    training_utils.assert_not_shuffled(dataset)

    with test.mock.patch.object(logging, 'warning') as mock_log:
      training_utils.verify_dataset_shuffled(dataset)
      self.assertRegexpMatches(
          str(mock_log.call_args),
          'input dataset `x` is not shuffled.')

    shuffled_dataset = dataset.shuffle(10)
    training_utils.verify_dataset_shuffled(shuffled_dataset)

class StandardizeWeightsTest(keras_parameterized.TestCase):

  def test_sample_weights(self):
    y = np.array([0, 1, 0, 0, 2])
    sample_weights = np.array([0.5, 1., 1., 0., 2.])
    weights = training_utils.standardize_weights(y, sample_weights)
    self.assertAllClose(weights, sample_weights)

  def test_class_weights(self):
    y = np.array([0, 1, 0, 0, 2])
    class_weights = {0: 0.5, 1: 1., 2: 1.5}
    weights = training_utils.standardize_weights(y, class_weight=class_weights)
    self.assertAllClose(weights, np.array([0.5, 1., 0.5, 0.5, 1.5]))

  def test_sample_weights_and_class_weights(self):
    y = np.array([0, 1, 0, 0, 2])
    sample_weights = np.array([0.5, 1., 1., 0., 2.])
    class_weights = {0: 0.5, 1: 1., 2: 1.5}
    weights = training_utils.standardize_weights(y, sample_weights,
                                                 class_weights)
    expected = sample_weights * np.array([0.5, 1., 0.5, 0.5, 1.5])
    self.assertAllClose(weights, expected)

  def test_dataset_with_class_weight(self):
    model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
    model.compile('rmsprop', 'mse')

    inputs = np.zeros((10, 3), np.float32)
    targets = np.zeros((10, 4), np.float32)
    dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
    dataset = dataset.repeat(100)
    dataset = dataset.batch(10)
    class_weight_np = np.array([0.25, 0.25, 0.25, 0.25])
    class_weight = dict(enumerate(class_weight_np))

    model.fit(
        dataset,
        epochs=1,
        steps_per_epoch=2,
        verbose=1,
        class_weight=class_weight)

if __name__ == '__main__':
  test.main()

max_stars_repo_name: fraudies/tensorflow

max_issues_repo_name: fraudies/tensorflow

max_forks_repo_name: fraudies/tensorflow


text:
import sys
import warnings
import numpy as np
import scipy.sparse as sp
from . import ch, utils
from .ch import pif
from .utils import timer

def clear_cache_single(node):
    node._cache['drs'].clear()
    if hasattr(node, 'dr_cached'):
        node.dr_cached.clear()

def vstack(x):
    x = [a if not isinstance(a, sp.linalg.interface.LinearOperator) else a.dot(np.eye(a.shape[1])) for a in x]
    return sp.vstack(x, format='csc') if any([sp.issparse(a) for a in x]) else np.vstack(x)
def hstack(x):
    x = [a if not isinstance(a, sp.linalg.interface.LinearOperator) else a.dot(np.eye(a.shape[1])) for a in x]
    return sp.hstack(x, format='csc') if any([sp.issparse(a) for a in x]) else np.hstack(x)

_giter = 0
class ChInputsStacked(ch.Ch):
    dterms = 'x', 'obj'
    terms = 'free_variables'

    def compute_r(self):
        if not hasattr(self, 'fevals'):
            self.fevals = 0
        self.fevals += 1
        return self.obj.r.ravel()

    def dr_wrt(self, wrt, profiler=None):
        '''
        Loop over free variables and delete cache for the whole tree after finished each one
        '''
        if wrt is self.x:
            jacs = []
            for fvi, freevar in enumerate(self.free_variables):
                tm = timer()
                if isinstance(freevar, ch.Select):
                    new_jac = self.obj.dr_wrt(freevar.a, profiler=profiler)
                    try:
                        new_jac = new_jac[:, freevar.idxs]
                    except:
                        # non-csc sparse matrices may not support column-wise indexing
                        new_jac = new_jac.tocsc()[:, freevar.idxs]
                else:
                    new_jac = self.obj.dr_wrt(freevar, profiler=profiler)

                pif('dx wrt {} in {}sec, sparse: {}'.format(freevar.short_name, tm(), sp.issparse(new_jac)))

                if self._make_dense and sp.issparse(new_jac):
                    new_jac = new_jac.todense()
                if self._make_sparse and not sp.issparse(new_jac):
                    new_jac = sp.csc_matrix(new_jac)

                if new_jac is None:
                    raise Exception(
                        'Objective has no derivative wrt free variable {}. '
                        'You should likely remove it.'.format(fvi))

                jacs.append(new_jac)
            tm = timer()
            utils.dfs_do_func_on_graph(self.obj, clear_cache_single)
            pif('dfs_do_func_on_graph in {}sec'.format(tm()))
            tm = timer()
            J = hstack(jacs)
            pif('hstack in {}sec'.format(tm()))
            return J

    def on_changed(self, which):
        global _giter
        _giter += 1
        if 'x' in which:
            pos = 0
            for idx, freevar in enumerate(self.free_variables):
                sz = freevar.r.size
                rng = np.arange(pos, pos+sz)
                if isinstance(self.free_variables[idx], ch.Select):
                    # Deal with nested selects
                    selects = []
                    a = self.free_variables[idx]
                    while isinstance(a, ch.Select):
                        selects.append(a.idxs)
                        a = a.a
                    newv = a.x.copy()
                    idxs = selects.pop()
                    while len(selects) > 0:
                        idxs = idxs[selects.pop()]
                    newv.ravel()[idxs] = self.x.r.ravel()[rng]
                    a.__setattr__('x', newv, _giter)
                elif isinstance(self.free_variables[idx].x, np.ndarray):
                    self.free_variables[idx].__setattr__('x', self.x.r[rng].copy().reshape(self.free_variables[idx].x.shape), _giter)
                else: # a number
                    self.free_variables[idx].__setattr__('x', self.x.r[rng], _giter)
                pos += sz

    @property
    def J(self):
        '''
        Compute Jacobian. Analyze dr graph first to disable unnecessary caching
        '''
        result = self.dr_wrt(self.x, profiler=self.profiler).copy()
        if self.profiler:
            self.profiler.harvest()
        return np.atleast_2d(result) if not sp.issparse(result) else result

def setup_sparse_solver(sparse_solver):
    _solver_fns = {
        'cg': lambda A, x, M=None : sp.linalg.cg(A, x, M=M, tol=1e-10)[0],
        'spsolve': lambda A, x : sp.linalg.spsolve(A, x)
    }
    if callable(sparse_solver):
        return sparse_solver
    elif isinstance(sparse_solver, str) and sparse_solver in list(_solver_fns.keys()):
        return _solver_fns[sparse_solver]
    else:
        raise Exception('sparse_solver argument must be either a string in the set (%s) or have the api of scipy.sparse.linalg.spsolve.' % ', '.join(list(_solver_fns.keys())))

def setup_objective(obj, free_variables, on_step=None, disp=True, make_dense=False):
    '''
    obj here can be a list of ch objects or a dict of label: ch objects. Either way, the ch
    objects will be merged into one objective using a ChInputsStacked. The labels are just used
    for printing out values per objective with each iteration. If make_dense is True, the
    resulting object with return a desne Jacobian
    '''
    # Validate free variables
    num_unique_ids = len(np.unique(np.array([id(freevar) for freevar in free_variables])))
    if num_unique_ids != len(free_variables):
        raise Exception('The "free_variables" param contains duplicate variables.')
    # Extract labels
    labels = {}
    if isinstance(obj, list) or isinstance(obj, tuple):
        obj = ch.concatenate([f.ravel() for f in obj])
    elif isinstance(obj, dict):
        labels = obj
        obj = ch.concatenate([f.ravel() for f in list(obj.values())])
    # build objective
    x = np.concatenate([freevar.r.ravel() for freevar in free_variables])
    obj = ChInputsStacked(obj=obj, free_variables=free_variables, x=x, make_dense=make_dense)
    # build callback
    def callback():
        if on_step is not None:
            on_step(obj)
        if disp:
            report_line = ['%.2e' % (np.sum(obj.r**2),)]
            for label, objective in sorted(list(labels.items()), key=lambda x: x[0]):
                report_line.append('%s: %.2e' % (label, np.sum(objective.r**2)))
            report_line = " | ".join(report_line) + '\n'
            sys.stderr.write(report_line)
    return obj, callback

class DoglegState(object):
    '''
    Dogleg preserves a great deal of state from iteration to iteration. Many of the things
    that we need to calculate are dependent only on this state (e.g. the various trust region
    steps, the current jacobian and the A & g that depends on it, etc.). Holding the state and
    the various methods based on that state here allows us to seperate a lot of the jacobian
    based calculation from the flow control of the optmization.

    There will be once instance of DoglegState per invocation of minimize_dogleg.
    '''
    def __init__(self, delta, solve):
        self.iteration = 0
        self._d_gn = None # gauss-newton
        self._d_sd = None # steepest descent
        self._d_dl = None # dogleg
        self.J = None
        self.A = None
        self.g = None
        self._p = None
        self.delta = delta
        self.solve = solve
        self._r = None
        self.rho = None
        self.done = False

    @property
    def p(self):
        '''p is the current proposed input vector'''
        return self._p
    @p.setter
    def p(self, val):
        self._p = val.reshape((-1, 1))

    # induce some certainty about what the shape of the steps are
    @property
    def d_gn(self):
        return self._d_gn
    @d_gn.setter
    def d_gn(self, val):
        if val is not None:
            val = val.reshape((-1, 1))
        self._d_gn = val

    @property
    def d_sd(self):
        return self._d_sd
    @d_sd.setter
    def d_sd(self, val):
        if val is not None:
            val = val.reshape((-1, 1))
        self._d_sd = val

    @property
    def d_dl(self):
        return self._d_dl
    @d_dl.setter
    def d_dl(self, val):
        if val is not None:
            val = val.reshape((-1, 1))
        self._d_dl = val

    @property
    def step(self):
        return self.d_dl.reshape((-1, 1))
    @property
    def step_size(self):
        return np.linalg.norm(self.d_dl)

    def start_iteration(self):
        self.iteration += 1
        pif('beginning iteration %d' % (self.iteration,))
        self.d_sd = (np.linalg.norm(self.g)**2 / np.linalg.norm(self.J.dot(self.g))**2 * self.g).ravel()
        self.d_gn = None

    @property
    def r(self):
        '''r is the residual at the current p'''
        return self._r
    @r.setter
    def r(self, val):
        self._r = val.copy().reshape((-1, 1))
        self.updateAg()

    def updateAg(self):
        tm = timer()
        pif('updating A and g...')
        JT = self.J.T
        self.A = JT.dot(self.J)
        self.g = JT.dot(-self.r).reshape((-1, 1))
        pif('A and g updated in %.2fs' % tm())

    def update_step(self):
        # if the Cauchy point is outside the trust region,
        # take that direction but only to the edge of the trust region
        if self.delta is not None and np.linalg.norm(self.d_sd) >= self.delta:
            pif('PROGRESS: Using stunted cauchy')
            self.d_dl = np.array(self.delta/np.linalg.norm(self.d_sd) * self.d_sd).ravel()
        else:
            if self.d_gn is None:
                # We only need to compute this once per iteration
                self.updateGN()
            # if the gauss-newton solution is within the trust region, use it
            if self.delta is None or np.linalg.norm(self.d_gn) <= self.delta:
                pif('PROGRESS: Using gauss-newton solution')
                self.d_dl = np.array(self.d_gn).ravel()
                if self.delta is None:
                    self.delta = np.linalg.norm(self.d_gn)
            else: # between cauchy step and gauss-newton step
                pif('PROGRESS: between cauchy and gauss-newton')
                # apply step
                self.d_dl = self.d_sd + self.beta_multiplier * (self.d_gn - self.d_sd)

    @property
    def beta_multiplier(self):
        delta_sq = self.delta**2
        diff = self.d_gn - self.d_sd
        sqnorm_sd = np.linalg.norm(self.d_sd)**2
        pnow = diff.T.dot(diff)*delta_sq + self.d_gn.T.dot(self.d_sd)**2 - np.linalg.norm(self.d_gn)**2 * sqnorm_sd
        return float(delta_sq - sqnorm_sd) / float((diff).T.dot(self.d_sd) + np.sqrt(pnow))

    def updateGN(self):
        tm = timer()
        if sp.issparse(self.A):
            self.A.eliminate_zeros()
            pif('sparse solve...sparsity infill is %.3f%% (hessian %dx%d)' % (100. * self.A.nnz / (self.A.shape[0] * self.A.shape[1]), self.A.shape[0], self.A.shape[1]))
            if self.g.size > 1:
                self.d_gn = self.solve(self.A, self.g).ravel()
                if np.any(np.isnan(self.d_gn)) or np.any(np.isinf(self.d_gn)):
                    from scipy.sparse.linalg import lsqr
                    warnings.warn("sparse solve failed, falling back to lsqr")
                    self.d_gn = lsqr(self.A, self.g)[0].ravel()
            else:
                self.d_gn = np.atleast_1d(self.g.ravel()[0]/self.A[0,0])
            pif('sparse solve...done in %.2fs' % tm())
        else:
            pif('dense solve...')
            try:
                self.d_gn = np.linalg.solve(self.A, self.g).ravel()
            except Exception:
                warnings.warn("dense solve failed, falling back to lsqr")
                self.d_gn = np.linalg.lstsq(self.A, self.g)[0].ravel()
            pif('dense solve...done in %.2fs' % tm())

    def updateJ(self, obj):
        tm = timer()
        pif('computing Jacobian...')
        self.J = obj.J
        if self.J is None:
            raise Exception("Computing Jacobian failed!")
        if sp.issparse(self.J):
            tm2 = timer()
            self.J = self.J.tocsr()
            pif('converted to csr in {}secs'.format(tm2()))
            assert(self.J.nnz > 0)
        elif ch.VERBOSE:
            nonzero = np.count_nonzero(self.J)
            pif('Jacobian dense with sparsity %.3f' % (nonzero/self.J.size))
        pif('Jacobian (%dx%d) computed in %.2fs' % (self.J.shape[0], self.J.shape[1], tm()))
        if self.J.shape[1] != self.p.size:
            raise Exception('Jacobian size mismatch with objective input')
        return self.J

    class Trial(object):
        '''
        Inside each iteration of dogleg we propose a step and check to see if it's actually
        an improvement before we accept it. This class encapsulates that trial and the
        testing to see if it is actually an improvement.

        There will be one instance of Trial per iteration in dogleg.
        '''
        def __init__(self, proposed_r, state):
            self.r = proposed_r
            self.state = state
            # rho is the ratio of...
            # (improvement in SSE) / (predicted improvement in SSE)
            self.rho = np.linalg.norm(state.r)**2 - np.linalg.norm(proposed_r)**2
            if self.rho > 0:
                with warnings.catch_warnings():
                    warnings.filterwarnings('ignore',category=RuntimeWarning)
                    predicted_improvement = 2. * state.g.T.dot(state.d_dl) - state.d_dl.T.dot(state.A.dot(state.d_dl))
                    self.rho /= predicted_improvement

        @property
        def is_improvement(self):
            return self.rho > 0

        @property
        def improvement(self):
            return (np.linalg.norm(self.state.r)**2 - np.linalg.norm(self.r)**2) / np.linalg.norm(self.state.r)**2

    def trial_r(self, proposed_r):
        return self.Trial(proposed_r, self)

    def updateRadius(self, rho, lb=.05, ub=.9):
        if rho > ub:
            self.delta = max(self.delta, 2.5*np.linalg.norm(self.d_dl))
        elif rho < lb:
            self.delta *= .25

def minimize_dogleg(obj, free_variables, on_step=None,
                     maxiter=200, max_fevals=np.inf, sparse_solver='spsolve',
                     disp=True, e_1=1e-15, e_2=1e-15, e_3=0., delta_0=None,
                     treat_as_dense=False):
    """"Nonlinear optimization using Powell's dogleg method.
    See Lourakis et al, 2005, ICCV '05, "Is Levenberg-Marquardt the
    Most Efficient Optimization for Implementing Bundle Adjustment?":
    http://www.ics.forth.gr/cvrl/publications/conferences/0201-P0401-lourakis-levenberg.pdf

    e_N are stopping conditions:
    e_1 is gradient magnatude threshold
    e_2 is step size magnatude threshold
    e_3 is improvement threshold (as a ratio; 0.1 means it must improve by 10%% at each step)

    maxiter and max_fevals are also stopping conditions. Note that they're not quite the same,
    as an iteration may evaluate the function more than once.

    sparse_solver is the solver to use to calculate the Gauss-Newton step in the common case
    that the Jacobian is sparse. It can be 'spsolve' (in which case scipy.sparse.linalg.spsolve
    will be used), 'cg' (in which case scipy.sparse.linalg.cg will be used), or any callable
    that matches the api of scipy.sparse.linalg.spsolve to solve `A x = b` for x where A is sparse.

    cg, uses a Conjugate Gradient method, and will be faster if A is sparse but x is dense.
    spsolve will be faster if x is also sparse.

    delta_0 defines the initial trust region. Generally speaking, if this is set too low then
    the optimization will never really go anywhere (to small a trust region to make any real
    progress before running out of iterations) and if it's set too high then the optimization
    will diverge immidiately and go wild (such a large trust region that the initial step so
    far overshoots that it can't recover). If it's left as None, it will be automatically
    estimated on the first iteration; it's always updated at each iteration, so this is treated
    only as an initialization.

    handle_as_dense explicitly converts all Jacobians of obj to dense matrices
    """

    solve = setup_sparse_solver(sparse_solver)
    obj, callback = setup_objective(obj, free_variables, on_step=on_step, disp=disp,
                                    make_dense=treat_as_dense)

    state = DoglegState(delta=delta_0, solve=solve)
    state.p = obj.x.r

    #inject profiler if in DEBUG mode
    if ch.DEBUG:
        from .monitor import DrWrtProfiler
        obj.profiler = DrWrtProfiler(obj)

    callback()
    state.updateJ(obj)
    state.r = obj.r

    def stop(msg):
        if not state.done:
            pif(msg)
        state.done = True

    if np.linalg.norm(state.g, np.inf) < e_1:
        stop('stopping because norm(g, np.inf) < %.2e' % e_1)
    while not state.done:
        state.start_iteration()
        while True:
            state.update_step()
            if state.step_size <= e_2 * np.linalg.norm(state.p):
                stop('stopping because of small step size (norm_dl < %.2e)' % (e_2 * np.linalg.norm(state.p)))
            else:
                tm = timer()
                obj.x = state.p + state.step
                trial = state.trial_r(obj.r)
                pif('Residuals computed in %.2fs' % tm())
                # if the objective function improved, update input parameter estimate.
                # Note that the obj.x already has the new parms,
                # and we should not set them again to the same (or we'll bust the cache)
                if trial.is_improvement:
                    state.p = state.p + state.step
                    callback()
                    if e_3 > 0. and trial.improvement < e_3:
                        stop('stopping because improvement < %.1e%%' % (100*e_3))
                    else:
                        state.updateJ(obj)
                        state.r = trial.r
                        if np.linalg.norm(state.g, np.inf) < e_1:
                            stop('stopping because norm(g, np.inf) < %.2e' % e_1)
                else:  # Put the old parms back
                    obj.x = ch.Ch(state.p)
                    obj.on_changed('x') # copies from flat vector to free variables
                # update our trust region
                state.updateRadius(trial.rho)
                if state.delta <= e_2*np.linalg.norm(state.p):
                    stop('stopping because trust region is too small')
            if state.done or trial.is_improvement or (obj.fevals >= max_fevals):
                break
        if state.iteration >= maxiter:
            stop('stopping because max number of user-specified iterations (%d) has been met' % maxiter)
        elif obj.fevals >= max_fevals:
            stop('stopping because max number of user-specified func evals (%d) has been met' % max_fevals)
    return obj.free_variables

max_stars_repo_name: JulesDoe/hmr

max_issues_repo_name: JulesDoe/hmr

max_forks_repo_name: JulesDoe/hmr

reason: import numpy,import scipy,from scipy


text:
import pandas as pd
import numpy as np
import torch.nn as nn
from torchvision import transforms
from torch.utils.data import Dataset
from collections import Counter
import torch
from PIL import Image
from skimage import io
#from data_util import TrainProtsDataset, ValProtsDataset, TestProtsDataset
from torch.utils.data import DataLoader
# from model
from tqdm import tqdm
import torch.optim as optim
from sklearn.metrics import f1_score
import torch.nn.functional as F
#from model import PretrainedResnet50