text:
import pygame
import sys
from pygame.locals import * # FOR KEYDOWN EVENT !!!
import numpy as np
import math
import random
# for creating the screen board
pygame.init()

STEP = 0
HEIGHT = 600
WIDTH = 600
LINE_WIDTH = 15
SQUARE_SIZE = 200
OFFSET = 55
CIRCLE_RADIUS = 60
CIRCLE_WIDTH = 15
X_LINE_WIDTH = 25
# TODO PLAYER ONE WINS COLORS AND PLAYER TWO WINS COLORS

# FOR FONTS
myfont = pygame.font.SysFont("monospace", 45)
# FOR COLORS
RED = (255, 0, 0)
BG_COLOR = (21, 181, 176)
LINE_COLOR = (23, 145, 135)
CIRCLE_COLOR = (249, 189, 192)
CROSS_COLOR = (251, 230, 152)
X_COLOR = (251, 230, 152)
DRAW_COLOR = (109, 236, 224)
# FOR BOARD ROWS AND COLUMNS
BOARD_ROWS = 3
BOARD_COLS = 3

# board
board = np.zeros((BOARD_ROWS, BOARD_COLS))

screen = pygame.display.set_mode( (WIDTH, HEIGHT) ) # for the screen background size
pygame.display.set_caption("TIC TAC TOE")
screen.fill( BG_COLOR ) # the background color
player1 = 1
player2 = 2

def draw_lines():
    # 1 horizontal
    pygame.draw.line( screen, LINE_COLOR, (0, 200), (600, 200), LINE_WIDTH )
    # 2 horizontal
    pygame.draw.line(screen, LINE_COLOR, (0, 400), (600, 400), LINE_WIDTH)

    # 1 vertical
    pygame.draw.line(screen, LINE_COLOR, (200, 0), (200, 600), LINE_WIDTH)
    # 2 vertical
    pygame.draw.line(screen, LINE_COLOR, (400, 0), (400, 600), LINE_WIDTH)

def draw_figures():
    for row in range(BOARD_ROWS):
        for col in range(BOARD_COLS):
            if board[row][col] == 1:
                pygame.draw.circle( screen, CIRCLE_COLOR, (int( col * 200 + 100), int( row * 200 + 100)), CIRCLE_RADIUS, CIRCLE_WIDTH )
            elif board[row][col] == 2:
                pygame.draw.line(screen, X_COLOR, (col * SQUARE_SIZE + OFFSET, row * SQUARE_SIZE + OFFSET),
                                 (col * SQUARE_SIZE + SQUARE_SIZE - OFFSET, row * SQUARE_SIZE + SQUARE_SIZE - OFFSET),
                                 X_LINE_WIDTH)
                pygame.draw.line(screen, X_COLOR,
                                 (col * SQUARE_SIZE + OFFSET, row * SQUARE_SIZE + SQUARE_SIZE - OFFSET),
                                 (col * SQUARE_SIZE + SQUARE_SIZE - OFFSET, row * SQUARE_SIZE + OFFSET), X_LINE_WIDTH)

def mark_squares(row, col, player): # fill the board by O or X
    board[row][col] = player

def available_square(row, col):
    return board[row][col] == 0 # if 0 means not filled yet

def is_board_full():
    return STEP == 9

def fake_check_win(board, player): # todo don't draw lines
    # 0 means continue 1 means gameover

    # vertical win check
    for col in range(BOARD_COLS):
        if board[0][col] == player and board[1][col] == player and board[2][col] == player:
            return True
    # horizontal win check
    for row in range(BOARD_ROWS):
        if board[row][0] == player and board[row][1] == player and board[row][2] == player:
            return True
    # asc diagonal win check
    if board[2][0] == player and board[1][1] == player and board[0][2] == player:
        return True
    # desc diagonal win check
    if board[0][0] == player and board[1][1] == player and board[2][2] == player:
        return True

    return False

def check_win(board, player): # todo draw lines
    # 0 means continue 1 means gameover
    gameover = 0
    # vertical win check
    for col in range(BOARD_COLS):
        if board[0][col] == player and board[1][col] == player and board[2][col] == player:
            draw_vertical_winning_line(col, player)
            gameover = 1
    # horizontal win check
    for row in range(BOARD_ROWS):
        if board[row][0] == player and board[row][1] == player and board[row][2] == player:
            draw_horizontal_winning_line(row, player)
            gameover = 1
    # asc diagonal win check
    if board[2][0] == player and board[1][1] == player and board[0][2] == player:
        draw_asc_diagonal(player)
        gameover = 1
    # desc diagonal win check
    if board[0][0] == player and board[1][1] == player and board[2][2] == player:
        draw_desc_diagonal(player)
        gameover = 1

    return gameover == 1

def draw_vertical_winning_line(col, player):
    posX = col * 200 + 100

    if player == 1:
        color = CIRCLE_COLOR
    elif player == 2:
        color = CROSS_COLOR

    pygame.draw.line( screen, color, (posX, 15), (posX, HEIGHT - 15), 15 )
def is_game_over(board):
    for row in range(BOARD_ROWS):
        for col in range(BOARD_COLS):
            if board[row][col] == 0:
                return False
    return True

def drop_piece(b_copy, row, col, player):
    b_copy[row][col] = player

def get_valid_locations(board):
    locations = []
    for r in range(3):
        for c in range(3):
          if board[r][c] == 0:
              locations.append((r, c))
    return locations

def score_position(board, player):
    score = 0

    ## Score Horizontal
    for row in range(BOARD_ROWS):
        if board[row][0] == player and board[row][1] == player and board[row][2] == player:
            score += 10
    ## Score Vertical
    for col in range(BOARD_COLS):
        if board[0][col] == player and board[1][col] == player and board[2][col] == player:
            score += 10
    ## Check negatively sloped diaganols
    if board[0][0] == player and board[1][1] == player and board[2][2] == player:
        score += 10
    ## Check positively sloped diaganols
    if board[0][2] == player and board[1][1] == player and board[2][0] == player:
        score += 10

    if player  == player1:
        opp_player = 2
    else:
        opp_player = 1

    # todo if player oop point 3 minus -80
    ## Score Horizontal
    for row in range(BOARD_ROWS):
        if board[row][0] == opp_player and board[row][1] == opp_player and board[row][2] == opp_player:
            score -= 5
    ## Score Vertical
    for col in range(BOARD_COLS):
        if board[0][col] == opp_player and board[1][col] == opp_player and board[2][col] == opp_player:
            score -= 5
    ## Check negatively sloped diaganols
    if board[0][0] == opp_player and board[1][1] == opp_player and board[2][2] == opp_player:
        score -= 5
    ## Check positively sloped diaganols
    if board[0][2] == opp_player and board[1][1] == opp_player and board[2][0] == opp_player:
        score -= 5

    print("player is " + str(player) + " " + "opp_player is " + str(opp_player))
    if score < 0:
        print("opp score "  + str(score) )
    return score

def minimax(board, depth, alpha, beta, maximizingPlayer): # todo False player1 True player2
    valid_locations = get_valid_locations(board) # todo write myself (is ok)
    is_gameover = is_game_over(board) # todo write nested loop myself
    # todo the ai first step put middle
    empty = 0
    middle = board[1][1]
    for row in range(BOARD_ROWS):
        for col in range(BOARD_COLS):
            if board[row][col] == 0:
                empty += 1
    if empty == 8 and middle == 2:
        return 1, 1, 100000000
    if depth == 0 or is_gameover:
        if is_gameover:
            print("is_gameover error")
            if fake_check_win(board, player2):# todo AI_PIECE to player2
                print("error 1")
                return (None, None, 10000000000000)
            elif fake_check_win(board, player1): # todo PLAYER_PIECE to player1
                print("error 2")
                return (None, None, -10000000000000)
            else: # Game is over, no more valid moves
                print("error 3")
                return (None, None, 0)
        else: # Depth is zero
            print("error 4")
            return (None, None, score_position(board, player2)) # todo figure why wrote like this

    if maximizingPlayer:
        value = -math.inf

        if valid_locations != None:
            tmp = random.choice(valid_locations)
            ans_row = tmp[0]
            ans_col = tmp[1]
            # todo below drop piece
            for square in valid_locations:
                row = square[0] # todo so we have valid locations and just go through it
                col = square[1]
                b_copy = board.copy() # todo important so that we don't affect the original board.
                drop_piece(b_copy, row, col, player2)
                new_score = minimax(b_copy, depth - 1, alpha, beta, False)[2] # get the score_position when depth is zero
                if new_score > value:
                    value = new_score
                    ans_col  = col
                    ans_row = row
                alpha = max(alpha, value)
                if alpha >= beta:
                    break
            return ans_row, ans_col, value
        else:
            return None, None, value

    else: # Minimizing player
        value = math.inf  # infinity numbers

        if valid_locations != None:
            tmp = random.choice(valid_locations)
            ans_row = tmp[0]
            ans_col = tmp[1]
            for square in valid_locations:
                row = square[0]  # todo so we have valid locations and just go through it
                col = square[1]
                b_copy = board.copy()
                drop_piece(b_copy, row, col, player1)
                new_score = minimax(b_copy, depth-1, alpha, beta, True)[2]
                if new_score < value:
                    value = new_score
                    ans_col = col
                    ans_row = row
                beta = min(beta, value)
                if alpha >= beta:
                    break
            return ans_row, ans_col, value
        else:
            return None, None, value

def draw_horizontal_winning_line(row, player):
    posY = row * 200 + 100

    if player == 1:
        color = CIRCLE_COLOR
    elif player == 2:
        color = CROSS_COLOR

    pygame.draw.line(screen, color, (15, posY), (WIDTH - 15, posY), 15)

def draw_asc_diagonal(player):
    if player == 1:
        color = CIRCLE_COLOR
    elif player == 2:
        color = CROSS_COLOR

    pygame.draw.line( screen, color, (15, HEIGHT - 15), (WIDTH - 15, 15), 15 )

def draw_desc_diagonal(player):
    if player == 1:
        color = CIRCLE_COLOR
    elif player == 2:
        color = CROSS_COLOR

    pygame.draw.line(screen, color, (15, 15), (WIDTH - 15, HEIGHT - 15), 15)

def restart():
    screen.fill(BG_COLOR)  # the background color
    draw_lines()
    for row in range(BOARD_ROWS):
        for col in range(BOARD_COLS):
            board[row][col] = 0

player = 1
game_over = False
draw_lines()
step = 0
# main loop
while True:
    if step == 9 and not game_over:
        label = myfont.render("DRAW", 1, DRAW_COLOR)
        screen.blit(label, (250, 200))
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            sys.exit()
        if event.type == KEYDOWN:
            if event.key == K_RETURN:
                step = 0
                game_over = False
                restart()

        if  player == 1 and event.type == pygame.MOUSEBUTTONDOWN and not game_over and step != 9:
            mouseX = event.pos[0] # x
            mouseY = event.pos[1] # y

            clicked_col = int(mouseX // 200)
            clicked_row = int(mouseY // 200)

            if available_square( clicked_row, clicked_col ):
                step += 1
                if player == 1:
                    mark_squares( clicked_row, clicked_col, 1)
                    if check_win(board, player):
                        game_over = True
                        label = myfont.render("Player 1 wins!!", 1, CIRCLE_COLOR)
                        screen.blit(label, (100, 200))
                    player = 2

    if step == 9 and not game_over:
        label = myfont.render("DRAW", 1, DRAW_COLOR)
        screen.blit(label, (250, 200))
    if player == 2 and not game_over and step != 9:
        print(step)
        # col = random.randint(0, COLUMN_COUNT-1)
        #col = pick_best_move(board, AI_PIECE)
        row, col, minimax_score = minimax(board, 4, -math.inf, math.inf, True)
        print(row, col, minimax_score)
        if available_square(row, col):
            step += 1
            pygame.time.wait(500)
            mark_squares( row, col, 2)

            if check_win(board, player):
                game_over = True
                label = myfont.render("AI wins!!", 1, CROSS_COLOR)
                screen.blit(label, (200, 200))
            player = 1

    draw_figures()
    pygame.display.update()


text:
"""
 Copyright 2018 Johns Hopkins University  (Author: Jesus Villalba)
 Apache 2.0  (http://www.apache.org/licenses/LICENSE-2.0)
"""
from __future__ import absolute_import
from __future__ import print_function
from __future__ import division
from six.moves import xrange

import numpy as np

from ..pdfs.plda import FRPLDA, SPLDA, PLDA

class PLDAFactory(object):
    """Class to  create PLDA objects."""

    @staticmethod
    def create_plda(plda_type, y_dim=None, z_dim=None, fullcov_W=True,
                    update_mu=True, update_V=True, update_U=True,
                    update_B=True, update_W=True, update_D=True,
                    floor_iD=1e-5,
                    name='plda', **kwargs):

        if plda_type == 'frplda':
            return FRPLDA(fullcov_W=fullcov_W,
                          update_mu=update_mu, update_B=update_B,
                          update_W=update_W, name=name, **kwargs)
        if plda_type == 'splda':
            return SPLDA(y_dim=y_dim, fullcov_W=fullcov_W,
                         update_mu=update_mu, update_V=update_V,
                         update_W=update_W, name=name, **kwargs)

        if plda_type == 'plda':
            return PLDA(y_dim=y_dim, z_dim=z_dim, floor_iD=floor_iD,
                        update_mu=update_mu, update_V=update_V,
                        update_U=update_U, update_D=update_D,
                        name=name, **kwargs)

    @staticmethod
    def load_plda(plda_type, model_file):
        if plda_type == 'frplda':
            return FRPLDA.load(model_file)
        elif plda_type == 'splda':
            return SPLDA.load(model_file)
        elif plda_type == 'plda':
            return PLDA.load(model_file)

    @staticmethod
    def filter_train_args(prefix=None, **kwargs):
        if prefix is None:
            p = ''
        else:
            p = prefix + '_'
        valid_args = ('plda_type', 'y_dim', 'z_dim',
                      'diag_W', 'no_update_mu', 'no_update_V', 'no_update_U',
                      'no_update_B', 'no_update_W', 'no_update_D', 'floor_iD',
                      'epochs', 'ml_md', 'md_epochs', 'name')
        d = dict((k, kwargs[p+k])
                 for k in valid_args if p+k in kwargs)
        neg_args1 = ('diag_W', 'no_update_mu', 'no_update_V', 'no_update_U',
                      'no_update_B', 'no_update_W', 'no_update_D')
        neg_args2 = ('fullcov_W', 'update_mu', 'update_V', 'update_U',
                      'update_B', 'update_W', 'update_D')

        for a,b in zip(ne_args1, neg_args2):
            d[b] = not d[a]
            del d[a]

        return d

    @staticmethod
    def add_argparse_train_args(parser, prefix=None):
        if prefix is None:
            p1 = '--'
            p2 = ''
        else:
            p1 = '--' + prefix + '-'
            p2 = prefix + '_'

        parser.add_argument(p1+'plda-type', dest=(p2+'plda_type'),
                            default='splda',
                            choices=['frplda', 'splda', 'plda'],
                            help='PLDA type')

        parser.add_argument(p1+'y-dim', dest=(p2+'y_dim'), type=int,
                            default=150,
                            help='num. of eigenvoices')
        parser.add_argument(p1+'z-dim', dest=(p2+'z_dim'), type=int,
                            default=400,
                            help='num. of eigenchannels')

        parser.add_argument(p1+'diag-W', dest=(p2+'diag_W'),
                            default=False, action='store_false',
                            help='use diagonal covariance W')
        parser.add_argument(p1+'no-update-mu', dest=(p2+'no_update_mu'),
                            default=False, action='store_true',
                            help='not update mu')
        parser.add_argument(p1+'no-update-V', dest=(p2+'no_update_V'),
                            default=False, action='store_true',
                            help='not update V')
        parser.add_argument(p1+'no-update-U', dest=(p2+'no_update_U'),
                            default=False, action='store_true',
                            help='not update U')

        parser.add_argument(p1+'no-update-B', dest=(p2+'no_update_B'),
                            default=False, action='store_true',
                            help='not update B')
        parser.add_argument(p1+'no-update-w', dest=(p2+'no_update_W'),
                            default=False, action='store_true',
                            help='not update W')
        parser.add_argument(p1+'no-update-d', dest=(p2+'no_update_D'),
                            default=False, action='store_true',
                            help='not update D')
        parser.add_argument(p1+'floor-id', dest=(p2+'floor_iD'), type=float,
                            default=1e-5,
                            help='floor for inverse of D matrix')

        parser.add_argument(p1+'epochs', dest=(p2+'epochs'), type=int,
                            default=40,
                            help='num. of epochs')
        parser.add_argument(p1+'ml-md', dest=(p2+'ml_md'),
                            default='ml+md',
                            choices=['ml+md', 'ml', 'md'],
                            help=('optimization type'))

        parser.add_argument('--md-epochs', dest='md_epochs', default=None,
                            type=int, nargs = '+',
                            help=('epochs in which we do MD, if None we do it in all the epochs'))

        parser.add_argument(p1+'name', dest=(p2+'name'),
                            default='plda',
                            help='model name')

    @staticmethod
    def filter_eval_args(prefix, **kwargs):
        if prefix is None:
            p = ''
        else:
            p = prefix + '_'
        valid_args = ('plda_type', 'model_file')
        return dict((k, kwargs[p+k])
                    for k in valid_args if p+k in kwargs)

    @staticmethod
    def add_argparse_eval_args(parser, prefix=None):
        if prefix is None:
            p1 = '--'
            p2 = ''
        else:
            p1 = '--' + prefix + '-'
            p2 = prefix + '_'

        parser.add_argument(p1+'plda-type', dest=(p2+'plda_type'),
                            default='splda',
                            choices=['frplda', 'splda', 'plda'],
                            help=('PLDA type'))
        parser.add_argument(p1+'model-file', dest=(p2+'model_file'), required=True,
                            help=('model file'))

ext: py

reason: import numpy


text:
from collections import defaultdict
import os
import pickle
import sys

import numpy as np

from rdkit import Chem

def pad(x_list, p):
    """Pad x_list with p."""
    len_max = max(map(len, x_list))
    pad_list = []
    for x in x_list:
        len_x = len(x)
        if (len_x < len_max):
            x += [p] * (len_max-len_x)
        pad_list.append(x)
    pad_array = np.array(pad_list)
    if (pad_array.dtype == 'int64'):
        return np.array(pad_array, np.int32)
    elif (pad_array.dtype == 'float64'):
        return np.array(pad_array, np.float32)

def make_atomlist(mol):
    atom_list = [atom_dict[a.GetSymbol()] for a in mol.GetAtoms()]
    return np.array(atom_list, np.int32)

def make_ijbonddict(mol):
    i_jbond_dict = defaultdict(lambda: [])
    for b in mol.GetBonds():
        i, j = b.GetBeginAtomIdx(), b.GetEndAtomIdx()
        bond = bond_dict[str(b.GetBondType())]
        i_jbond_dict[i].append((j, bond))
        i_jbond_dict[j].append((i, bond))
    return i_jbond_dict

def make_adjalist(mol):
    adja_mat = Chem.GetAdjacencyMatrix(mol)
    adja_list = [list(np.where(adja == 1)[0]) for adja in adja_mat]
    return adja_list

def Weisfeiler_Lehman(atom_list, i_jbond_dict, radius):
    """Extract the r-radius vertices and r-radius edges
    from a graph using WeisfeilerLehman-like algorithm."""

    if (radius == 0):
        vertex_list = atom_list
        edge_list = []
        for i, j_bond in i_jbond_dict.items():
            edge_list.append([bond for j, bond in j_bond])

    else:
        for _ in range(radius):

            """r-radius vertex."""
            vertex_list = []
            for i, j_bond in i_jbond_dict.items():
                v = (atom_list[i], tuple(sorted([(atom_list[j], bond)
                     for j, bond in j_bond])))
                v = vertex_dict[v]
                vertex_list.append(v)

            """r-radius edge."""
            edge_list = []
            for i, j_bond in i_jbond_dict.items():
                e_list = []
                for j, bond in j_bond:
                    e = (bond, tuple(sorted((atom_list[i], atom_list[j]))))
                    e = edge_dict[e]
                    e_list.append(e)
                edge_list.append(e_list)

            """Update atom_list and i_jbond_dict."""
            atom_list = vertex_list
            i_jedge_dict = defaultdict(lambda: [])
            for i, j_bond in i_jbond_dict.items():
                for k, (j, bond) in enumerate(j_bond):
                    i_jedge_dict[i].append((j, edge_list[i][k]))
            i_jbond_dict = i_jedge_dict

    return np.array(vertex_list, np.int32), pad(edge_list, -1)

def make_vertex_(adja_list):
    """Make vertex_, which is a vertex matrix corresponding to an edge matrix
    to efficiently compute an edge transition function
    introduced in our GNN."""
    vertex_ = [[i for a in adja] for i, adja in enumerate(adja_list)]
    return pad(vertex_, -1)

def make_sequence(sequence, ngram):
    """Split a sequence into n-gram words."""
    word_list = [sequence[i:i+ngram] for i in range(len(sequence)-ngram+1)]
    sequence = [word_dict[word] for word in word_list]
    return np.array(sequence, np.int32)

def pickle_dump(dictionary, file_name):
    pickle.dump(dict(dictionary), open(file_name, 'wb'))

if __name__ == "__main__":

    DATASET, radius, ngram = sys.argv[1:]
    radius, ngram = map(int, [radius, ngram])
    directory = '../dataset/' + DATASET + '/original/'
    data_file = open(directory + 'smiles_sequence_interaction.txt', 'r')

    atom_dict = defaultdict(lambda: len(atom_dict))
    bond_dict = defaultdict(lambda: len(bond_dict))
    vertex_dict = defaultdict(lambda: len(vertex_dict))
    edge_dict = defaultdict(lambda: len(edge_dict))
    word_dict = defaultdict(lambda: len(word_dict))

    Vertex, Edge, Adjacency, Vertex_ = [], [], [], []
    Sequence = []
    Interaction = []

    no, n_positive, n_negative = 0, 0, 0

    for data in data_file:

        smiles, sequence, interaction = data.strip().split(' ')

        if interaction == '1':
            n_positive += 1
        elif interaction == '0':
            n_negative += 1

        """Atom list."""
        mol = Chem.MolFromSmiles(smiles)
        atom_list = make_atomlist(mol)

        """Note that we exclude '.' data in smiles."""
        if ('.' not in smiles):
            no += 1
            print('Sample:', no)

            """Bond dictionary."""
            i_jbond_dict = make_ijbonddict(mol)

            """The r-radius vertex and edge."""
            if (len(atom_list) == 1):
                vertex = np.array(atom_list, np.int32)
                edge = np.array([[-1]], np.int32)
                adja_list = [[-1]]
            else:
                vertex, edge = Weisfeiler_Lehman(
                               atom_list, i_jbond_dict, radius)
                adja_list = make_adjalist(mol)
            Vertex.append(vertex)
            Edge.append(edge)

            """Vertex_."""
            vertex_ = make_vertex_(adja_list)
            Vertex_.append(vertex_)

            """Pad the adjacency list."""
            adja = pad(adja_list, -1)
            Adjacency.append(adja)

            """Split the sequence."""
            sequence = make_sequence(sequence, ngram)
            Sequence.append(sequence)

            """Label (interact or not)."""
            interaction = np.array([int(interaction)], np.int32)
            Interaction.append(interaction)

    """The number of WL vertices and edges."""
    if (radius == 0):
        vertex_dict, edge_dict = atom_dict, bond_dict
    n_vertex, n_edge = len(vertex_dict), len(edge_dict)
    n_word = len(word_dict)

    """Save dataset."""
    radius, ngram = str(radius), str(ngram)
    directory = ('../dataset/' + DATASET + '/input/radius' +
                 radius + '_ngram' + ngram + '/')
    if not os.path.isdir(directory):
        os.mkdir(directory)
    np.save(directory + 'vertex', Vertex)
    np.save(directory + 'edge', Edge)
    np.save(directory + 'adjacency', Adjacency)
    np.save(directory + 'vertex_', Vertex_)
    np.save(directory + 'sequence', Sequence)
    np.save(directory + 'interaction', Interaction)
    np.save(directory + 'n_vertex', n_vertex)
    np.save(directory + 'n_edge', n_edge)
    np.save(directory + 'n_word', n_word)

    """Save dictionaries (for analysis)."""
    pickle_dump(atom_dict, directory + 'atom_dict.pickle')
    pickle_dump(bond_dict, directory + 'bond_dict.pickle')
    pickle_dump(vertex_dict, directory + 'vertex_dict' + '.pickle')
    pickle_dump(edge_dict, directory + 'edge_dict' + '.pickle')
    pickle_dump(word_dict, directory + 'word_dict' + '.pickle')

    print('-'*30)
    print('The preprocess is finished!\nDataset information is as follows.')
    print('Dataset: ', DATASET)
    print('Radius: ', radius)
    print('N-gram: ', ngram)
    print('The number of r-radius vertices: ', n_vertex)
    print('The number of r-radius edges: ', n_edge)
    print('The number of n-gram: ', n_word)
    print('The number of positive samples: ', n_positive)
    print('The number of negative samples: ', n_negative)
    print('-'*30)


text:
# import the pyabf module from this development folder
import os
import sys
PATH_HERE = os.path.abspath(os.path.dirname(__file__))
PATH_SRC = os.path.abspath(PATH_HERE+"/../../src/")
PATH_DATA = os.path.abspath(PATH_HERE+"/../../data/abfs/")
sys.path.insert(0, PATH_SRC)  # for importing
sys.path.append("../../src/")  # for your IDE
import pyabf
import glob

import matplotlib.pyplot as plt
import numpy as np
import warnings

def populateEpochInfo(abf, sweep=0, channel=0):
    """
    Provide easy-to-use lists of command values, types, and times.
    """

    warnings.warn("DONT USE THIS")

    # TODO: support more epoch types

    # TODO: create epochs for ABF1 files
    if abf.abfFileFormat != 2:
        raise NotImplementedError

    # populate epoch arrays with values appropriate for pre-sweep
    epochPoints = [0]
    epochValues = [abf.holdingCommand[channel]]
    epochTypes = [1]

    # the epoch doesn't actually start at the start of the sweep
    position = int(abf.sweepPointCount/64)

    # epochs are in a flat list repeated by channel
    epochCount = len(abf._epochPerDacSection.nEpochType)
    epochsOfThisChannel = np.arange(epochCount/abf.channelCount)
    epochsOfThisChannel += epochCount*channel

    #epochsOfThisChannel = range(len(abf._epochPerDacSection.nEpochType))

    # step through each epoch and track the command value and time position
    for epoch in epochsOfThisChannel:
        epoch = int(epoch)
        epochType = abf._epochPerDacSection.nEpochType[epoch]
        thisT = abf._epochPerDacSection.lEpochInitDuration[epoch]
        dT = abf._epochPerDacSection.lEpochDurationInc[epoch] * sweep
        thisC = abf._epochPerDacSection.fEpochInitLevel[epoch]
        dC = abf._epochPerDacSection.fEpochLevelInc[epoch] * sweep
        dClast = abf._epochPerDacSection.fEpochLevelInc[epoch] * (sweep-1)
        position += thisT+dT

        # add this epoch to the list
        epochTypes.append(epochType)
        epochValues.append(thisC+dC)
        epochPoints.append(position)

        # determine if the pre-command is holding or holdover from last sweep
        if epoch == 0 and abf._dacSection.nInterEpisodeLevel[channel]:
            epochValues[0] = thisC+dClast

    # add an after-epoch value (holding command or command holdover)
    epochTypes.append(1)
    epochPoints.append(position)
    epochValues.append(abf.holdingCommand[channel])
    if abf._dacSection.nInterEpisodeLevel[channel]:
        epochValues[-1] = thisC+dC

    # add a point to simulate the end of the sweep
    epochTypes.append(1)
    epochPoints.append(abf.sweepPointCount-1)
    epochValues.append(epochValues[-1])

    for epochType in epochTypes:
        if not epochType in [0, 1]:
            msg = f"epoch type {epochType} unsupported"
            warnings.warn(msg)

    #print(epochTypes, epochPoints, epochValues)

    return [epochPoints, epochValues]

if __name__ == "__main__":
    # abf = pyabf.ABF(PATH_DATA+"/171116sh_0013.abf")  # ramp
    # abf = pyabf.ABF(PATH_DATA+"/2018_04_13_0016b_modified.abf") # delta t
    # abf = pyabf.ABF(PATH_DATA+"/14o16001_vc_pair_step.abf")  # delta c
    abf = pyabf.ABF(PATH_DATA+"/18702001-step.abf")  # complex step

    print("DONE")

max_stars_repo_path: dev/python/2018-07-02 epochs.py

max_stars_repo_name: konung-yaropolk/pyABF

max_issues_repo_path: dev/python/2018-07-02 epochs.py

max_issues_repo_name: konung-yaropolk/pyABF

max_forks_repo_path: dev/python/2018-07-02 epochs.py

max_forks_repo_name: konung-yaropolk/pyABF


text:
"""
Copyright (C) 2019 University of Massachusetts Amherst.
This file is part of "expLinkage"
http://github.com/iesl/expLinkage
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import logging
from pathlib import Path
import  math
import numpy as np
import torch
import itertools
from utils.Config import Config

def create_logger(config, logFile=None, currLogger=None):

	assert isinstance(config, Config)
	if currLogger is not None:
		for handler in currLogger.handlers[:]:
			currLogger.removeHandler(handler)

	if logFile is not None:
		config.logFile = logFile

	l = logging.getLogger("new_logger")
	fileHandler = logging.FileHandler("{}/{}".format(config.resultDir, config.logFile), mode='a')
	l.setLevel(logging.DEBUG)
	l.addHandler(fileHandler)

	if config.logConsole: # Add console to logger
		console = logging.StreamHandler()
		console.setLevel(logging.INFO)
		l.addHandler(console)

	newLogger = logging.getLogger("new_logger")
	return newLogger

def load_canopy_data_splits(config):

	canopyData 		= read_canopy_data(config.dataDir)
	allCanopyIds 	= list(canopyData.keys())

	numTrain 	= int(math.ceil(len(allCanopyIds) * config.trainFrac))
	numTest 	= int(math.ceil(len(allCanopyIds) * config.testFrac))
	numDev 		= int(math.ceil(len(allCanopyIds) * config.devFrac))

	if config.shuffleData:
		np.random.shuffle(allCanopyIds)

	trainCanopyIds 	= allCanopyIds[:numTrain]
	testCanopyIds 	= allCanopyIds[numTrain:numTrain + numTest]
	devCanopyIds 	= allCanopyIds[numTrain + numTest: numTrain + numTest + numDev]

	trainCanopies 	= {id: canopyData[id] for id in trainCanopyIds}
	testCanopies 	= {id: canopyData[id] for id in testCanopyIds}
	devCanopies 	= {id: canopyData[id] for id in devCanopyIds}

	return trainCanopies, testCanopies, devCanopies

def load_vec_data_splits(config):

	if "face" in config.clusterFile:
		clusters = read_clusters(config.clusterFile)
	elif "synthetic" in config.clusterFile:
		clusters = read_clusters_synth(config.clusterFile)
	else:
		clusters = read_clusters_synth(config.clusterFile)

	while True:
		trainCanopies, devCanopies, testCanopies = split_vec_data(config=config, clusters=clusters)
		if not config.shuffleData:
			print("\n\nNot checking is split is balanced or not \n\n")
			break

		if "pathbased" in config.clusterFile or "synthetic" in config.clusterFile:
			break

		if eval_split_quality(trainCanopies, testCanopies, devCanopies):
			break

	return trainCanopies, testCanopies, devCanopies

def split_vec_data(config, clusters):
	''' Puts trainFrac fraction of clustres in trainClusters and remaining clusters in testClusters, but each
	  but in batches where each batch has testFrac fraction of clusters.'''

	trainFrac = config.trainFrac
	testFrac = config.testFrac
	devFrac = config.devFrac

	numTotal = len(clusters)
	numTrain = math.ceil(trainFrac * numTotal)
	numTest = math.ceil(testFrac * numTotal)
	if isinstance(devFrac, float):
		numDev = math.ceil(devFrac * numTotal)
	else:
		numDev = 0

	allCids = list(clusters.keys())
	if config.shuffleData:
		np.random.shuffle(allCids)

	########################## Create trainCanopy ###################################
	trainCids = allCids[:numTrain]
	trainClusters = {cid: clusters[cid] for cid in trainCids}
	trainPoints = {}
	pid = 0
	for cid in trainClusters:
		for point in trainClusters[cid]:
			trainPoints[pid] = (point, cid)
			pid += 1

	trainCanopies = [{"clusters": trainClusters, "points": trainPoints}]

	########################## Create devCanopy ###################################
	if numDev > 0:
		devCids = allCids[numTrain:numTrain + numDev]
		devClusters = {cid: clusters[cid] for cid in devCids}
		devPoints = {}
		pid = 0
		for cid in devClusters:
			for point in devClusters[cid]:
				devPoints[pid] = (point, cid)
				pid += 1

		devCanopies = [{"clusters": devClusters, "points": devPoints}]
	else:
		devCanopies = []

	########################## Create testCanopy ###################################
	testCanopies = []
	testStartCid = numTrain + numDev
	testEndCid = testStartCid + numTest
	numTestCanopies = 0
	while testStartCid < len(allCids):
		numTestCanopies += 1

		testCids = allCids[testStartCid:testEndCid]
		testClusters = {cid: clusters[cid] for cid in testCids}
		pid = 0
		testPoints = {}
		for cid in testClusters:
			for point in testClusters[cid]:
				testPoints[pid] = (point, cid)
				pid += 1

		canopy = {"clusters": testClusters, "points": testPoints}
		testCanopies.append(canopy)
		testStartCid += numTest
		testEndCid += numTest

	return trainCanopies, devCanopies, testCanopies

def eval_split_quality(trainCanopies, testCanopies, devCanopies):

	avgDist = {"train": 0, "test": 0, "dev": 0}
	for canopy in trainCanopies:
		centroid = []
		for cid in canopy["clusters"]:
			centroid.append(np.mean(canopy["clusters"][cid], axis=0))

		avgDist["train"] = []
		for c1, c2 in itertools.combinations(centroid, 2):
			avgDist["train"] += [np.linalg.norm(c1 - c2)]
		avgDist["train"] = np.mean(avgDist["train"])

	for canopy in devCanopies:
		centroid = []
		for cid in canopy["clusters"]:
			centroid.append(np.mean(canopy["clusters"][cid], axis=0))

		avgDist["dev"] = []
		for c1, c2 in itertools.combinations(centroid, 2):
			avgDist["dev"] += [np.linalg.norm(c1 - c2)]
		avgDist["dev"] = np.mean(avgDist["dev"])

	for canopy in testCanopies:
		centroid = []
		for cid in canopy["clusters"]:
			centroid.append(np.mean(canopy["clusters"][cid], axis=0))

		avgDist["test"] = []
		for c1, c2 in itertools.combinations(centroid, 2):
			avgDist["test"] += [np.linalg.norm(c1 - c2)]
		avgDist["test"] = np.mean(avgDist["test"])

	print("{}\n".format(avgDist))
	if abs(avgDist["dev"] - avgDist["test"]) < 0.05 * min(avgDist["dev"], avgDist["test"]):
		return True

	return False

def read_clusters_synth(filename):
	'''

	:param filename: Name of file with data
			Format of file: <x1> <x2> <clusterId>
	:return:
	clusters: Dictionaty with key as clusterIds, and value as list of points in that cluster
	'''
	clusters	= {}
	with open(filename,"r") as reader:
		for line in reader:
			splitLine  	= line.strip().split()
			point 		= tuple(float(x) for x in splitLine[0:2]) # First 2 numbers are floats and are coordinates of point
			cid		 	= int(splitLine[2]) # 3rd value contain cluster of this point
			if cid not in clusters:
				clusters[cid] = []

			clusters[cid].append(point)

	return clusters

def read_clusters(filename):
	'''

	:param filename: Name of file with data
			Format of file: <point_id> <cluster_id> <dim1> <dim2> ....
	:return:
	clusters: Dictionaty with key as clusterIds, and value as list of points in that cluster
	'''

	clusters = {}
	with open(filename, "r") as reader:
		for line in reader:
			splitLine = line.strip().split()
			point = tuple(float(x) for x in splitLine[2:])
			cid 	= 	splitLine[1] # 2nd value contain cluster of this point
			pointId = 	splitLine[0] # 1st value contains id of he point

			if cid not in clusters:
				clusters[cid] = []

			clusters[cid].append(point)

	return clusters

def read_canopy_data(dataDir):
	'''

	:param dataDir: DataDir has folder for each canopy, and each folder contains pairFeatures.csv and gtClusters.tsv
	:return: canopyData: Dictionary with canopyId as keys, and
						canopyData[id] is  in turn a dictionary with keys
							pidToCluster : Dict mapping each point to its cluster
							clusterToPids: Dict mapping each cluster to list of ids of points in ti
							pairFeatures : Dict with (pid1,pid2) as key and feature vector over pid1 and pid2 as value

	'''

	canopyData = {}
	folderList = [str(f) for f in Path(dataDir).glob("*") if f.is_dir()]
	print("Number of files:{}".format(len(folderList)))

	for ctr, folder in enumerate(folderList):
		print("processing folder:{}\t{}".format(ctr,folder))
		pidToCluster = {}
		pidToIntPid = {}  # Maps each point Id in original file to an integer ID in range(0,numPoints-1) so that I can use them as indices
		intPidCtr 	= 0
		with open("{}/gtClusters.tsv".format(folder)) as gtFile:
			for line in gtFile:
				line = line.strip().split()
				pid, cid = int(line[0]), int(line[1])
				if pid not in pidToIntPid:
					pidToIntPid[pid] = intPidCtr
					intPidCtr += 1

				intPid = pidToIntPid[pid]
				pidToCluster[intPid] = cid

		assert intPidCtr == len(pidToCluster)
		clusterToPids = {}
		for pid in pidToCluster:
			cid = pidToCluster[pid]
			try:
				clusterToPids[cid].append(pid)
			except:
				clusterToPids[cid] = [pid]

		pairFeatures = {}
		with open("{}/pairFeatures.csv".format(folder)) as featFile:
			for line in featFile:
				line = line.strip().split(",")
				featureVec = [float(x) for x in line[2:-1]]
				pid1, pid2 = int(line[0]), int(line[1])

				# Replace pids read from file with their corresponding mapped ids
				pid1, pid2 = pidToIntPid[pid1], pidToIntPid[pid2]

				if pid1 > pid2:
					pid1, pid2 = pid2, pid1

				pairFeatures[(pid1, pid2)] = (featureVec)
				assert (pid1 <= pid2)
				if line[-1] == "1":
					assert (pidToCluster[pid1] == pidToCluster[pid2])
				elif line[-1] == "0":
					assert (pidToCluster[pid1] != pidToCluster[pid2])
				else:
					print(line)
					raise Exception("Unexcepted end token in line. Expected 1 or 0")

		canopyId = folder.split("/")[-1]
		canopyData[canopyId] = {"pidToCluster": pidToCluster, "clusterToPids": clusterToPids,
								"pairFeatures": pairFeatures}
	return canopyData

def get_filename_list(parameters, template):
	filenameList = [template]
	paramList = list(parameters.keys())

	for paramCtr, param in enumerate(paramList):
		updateFilenameList = []
		for template in filenameList:
			if isinstance(parameters[param],list):
				for paramValue in parameters[param]:
					dict = {param: paramValue}
					for restParam in paramList[paramCtr + 1:]:
						dict[restParam] = "{" + str(restParam) + "}"
					filename = template.format(**dict)
					updateFilenameList.append(filename)
			else:
				dict = {param: parameters[param]}
				for restParam in paramList[paramCtr + 1:]:
					dict[restParam] = "{" + str(restParam) + "}"
				filename = template.format(**dict)
				updateFilenameList.append(filename)

		filenameList = updateFilenameList

	# for filename in filenameList:
	# 	print(filename)
	return filenameList

def generate_statistics(trainer):

	# assert isinstance(trainer, PairFeatureTrainer)

	microAvgNumPtsInCluster = {"all":[]}
	avgNumPtsInCluster 	= {"all":[]}
	numSingleton	 	= {"all":[]}
	fracSingleton		= {"all":[]}
	numPoints 			= {"all":[]}
	numClusters 		= {"all":[]}
	for canopyId in trainer.trainCanopies:
		canopy = trainer.trainCanopies[canopyId]
		ptsInCluster = [len(canopy["clusterToPids"][cid]) for cid in canopy["clusterToPids"]]

		avgNumPtsInCluster[canopyId] = np.mean(ptsInCluster)
		numClusters[canopyId] 	= len(canopy["clusterToPids"])

		numSingleton[canopyId]  = sum([1 for cid in canopy["clusterToPids"] if len(canopy["clusterToPids"][cid]) == 1])
		numPoints[canopyId] = sum(ptsInCluster)

		microAvgNumPtsInCluster["all"] += ptsInCluster
		avgNumPtsInCluster["all"] += [avgNumPtsInCluster[canopyId]]
		numClusters["all"] += [numClusters[canopyId]]
		numSingleton["all"] += [numSingleton[canopyId]]
		numPoints["all"] += [numPoints[canopyId]]
		fracSingleton["all"] += [float(numSingleton[canopyId])/numPoints[canopyId]]

	trainer.logger.info("Macro:Average Number of Points in each cluster:\t{:.3f}\t{:.3f}".format( np.mean( avgNumPtsInCluster["all"]), np.std( avgNumPtsInCluster["all"] ) ))
	trainer.logger.info("Micro:Average Number of Points in each cluster:\t{:.3f}\t{:.3f}".format( np.mean( microAvgNumPtsInCluster["all"]), np.std( microAvgNumPtsInCluster["all"] ) ))
	trainer.logger.info("Macro:Average Number of Singleton in canopies :\t{:.3f}\t{:.3f}".format( np.mean( numSingleton["all"]), np.std( numSingleton["all"] ) ))
	trainer.logger.info("Micro:Average Number of Singleton in canopies :\t{:.3f}\t{:.3f}".format( np.mean( fracSingleton["all"]), np.std( fracSingleton["all"] ) ))
	trainer.logger.info("Average Number of Clusters :\t{:.3f}\t{:.3f}".format( np.mean( numClusters["all"]), np.std( numClusters["all"] ) ))
	trainer.logger.info("Average Number of Points :\t{:.3f}\t{:.3f}".format( np.mean( numPoints["all"]), np.std( numPoints["all"] ) ))
	trainer.logger.info("Average Number of Points in each cluster:\t{:.3f}\t{:.3f}".format( np.mean( avgNumPtsInCluster["all"]), np.std( avgNumPtsInCluster["all"] ) ))
	# plt.clf()
	# plt.xscale('log')
	# plt.hist(microAvgNumPtsInCluster["all"],int(np.max(microAvgNumPtsInCluster["all"])))
	# plt.savefig("{}/clusterSizes.png".format(trainer.config.resultDir))
	#
	# plt.close()
	# plt.clf()
	# plt.xscale('log')
	# plt.hist(microAvgNumPtsInCluster["all"],100)
	# plt.savefig("{}/clusterSizes_100Bins.png".format(trainer.config.resultDir))
	# plt.close()
	#
	# plt.clf()
	# plt.xscale('log')
	# plt.hist(microAvgNumPtsInCluster["all"],10)
	# plt.savefig("{}/clusterSizes_10Bins.png".format(trainer.config.resultDir))
	# plt.close()

def calc_batch_size(numPoints, pointDim):
	'''
	# Find allocated GPU Memory and uses that to estimate largest batch size that it can probably support
	# Assumes that 10GB is total memory available for use
	:param numPoints: Number of points for which we want to compute adjacency matrix
	:param pointDim: Dimension of each point
	:return: batchSize: Largest value of m such that numPoints x m dimensional matrix can be created of pointDim
						dimensional points
	'''

	MAX_AVAIL_MB = 10* (2**10)

	memAllocMB = torch.cuda.memory_allocated() / (2 ** 20)
	memAvailMB = (MAX_AVAIL_MB - memAllocMB)  # ASSUMING TOTAL OF 10 GB memory is available for allocation on GPU

	batchSize = int((2 ** 18) * memAvailMB / (numPoints *pointDim))

	return  batchSize

max_stars_repo_name: iesl/expLinkage

max_issues_repo_name: iesl/expLinkage

max_forks_repo_name: iesl/expLinkage


text:
import numpy as np
import numpy.random as npr
import matplotlib.pyplot as plt
import time
import jax
from jax.api import jit, grad
from jax.scipy.special import logsumexp
from jax.flatten_util import ravel_pytree
from jax import lax
import jax.numpy as jnp
import datasets
import wandb
import log_handler

# Functions which help with basic use of all architectures
def init_random_params(scale, layer_sizes, seed):
  np.random.seed(seed)
  return [(scale * np.random.randn(m, n), scale * np.random.randn(n))
          for m, n, in zip(layer_sizes[:-1], layer_sizes[1:])]

# Computes the ReLU activation function
def relu(x):
    return jnp.maximum(x, 0)

@jit
def predict(params, inputs):
  activations = inputs
  for w, b in params[:-1]:
    outputs = jnp.dot(activations, w) + b
    activations = jnp.tanh(outputs)

  final_w, final_b = params[-1]
  logits = jnp.dot(activations, final_w) + final_b
  return logits - logsumexp(logits, axis=1, keepdims=True)

@jit
def accuracy(params, batch):
  inputs, targets = batch
  target_class = jnp.argmax(targets, axis=1)
  predicted_class = jnp.argmax(predict(params, inputs), axis=1)
  return jnp.mean(predicted_class == target_class)

# Functions which computes the Power Method using the Efficient Hessian Multiplication
@jit
def params_matmul(params, batch, const_vector):
  grads = grad(loss)(params, batch)
  grads_flat, _ = ravel_pytree(grads)
  return jnp.dot(const_vector.T, grads_flat)

gpus = jax.devices('gpu')
cpus = jax.devices('cpu')
jit_QR = jax.jit(jnp.linalg.qr, device=gpus[1])
jit_grad = jax.jit(grad(params_matmul), device=gpus[1])
jit_zeros = jax.jit(jnp.zeros, static_argnums=[0], device=gpus[1])
jit_error = jax.jit(lambda Q, Q_prev: ((jnp.abs(Q) - jnp.abs(Q_prev)) ** 2).sum(), device=gpus[1])

def param_simultaneous_power_iteration(params, k, batch, init_Q):
    Q, _ = jnp.linalg.qr(init_Q)
    Q_prev = Q
    for i in range(100):
        Q, R = update_power_iteration(Q, params, batch, Q.shape)
        # can use other stopping criteria as well
        err = jit_error(Q, Q_prev)
        Q_prev = np.copy(Q)
        if err < 0.1:
            break
    return jnp.abs(jnp.diag(R)), Q

def update_power_iteration(Q, params, batch, z_shape):
    Z = jit_zeros(z_shape)
    for j in range(Q.shape[1]):
        update_vec = jit_grad(params, batch, Q[:,j])
        update_vec_ravel = ravel_pytree(update_vec)[0]
        Z = jax.ops.index_update(Z, jax.ops.index[:,j], update_vec_ravel)
    Q, R = jit_QR(Z)
    return Q,R

update_power_iteration = jax.jit(update_power_iteration, static_argnums=[3], device=cpus[0])

# Functions for the different possible pieces of a loss function
@jit
def loss(params, batch):
  inputs, targets = batch
  preds = predict(params, inputs)
  return -jnp.mean(jnp.sum(preds * targets, axis=1))

@jit
def mahalanobis_dist(params, batch, params_constant_ravel):
  inputs, targets = batch
  grads = grad(loss)(params, batch)
  grads_flat, _ = ravel_pytree(grads)
  return jnp.dot(params_constant_ravel.T, grads_flat)

#@jit
def param_volume(params, batch):
  eig_vals, _ = param_simultaneous_power_iteration(params, wandb.config.num_eigs_keep, batch)
  return(-jnp.log(jnp.prod(eig_vals)))

@jit
def l2_regularizer(params):
  flat_params, unflatten = ravel_pytree(params)
  return jnp.dot(flat_params.T, flat_params)

# Run experiments
if __name__ == "__main__":
  wandb.init()

  # Function which updates model parameters without regularization
  @jit
  def update_standard(params, batch, epoch):#, init_Q):
    grads = grad(loss)(params, batch)
    return [(w - step_size * dw, b - step_size * db) for (w, b), (dw, db) in zip(params, grads)]

  # Function which updates model parameters with mahalanobis dist regularizer
  @jit
  def update_mahal(params, batch, epoch):#, init_Q):
    const_flat_params, unflattener = ravel_pytree(params)
    grads = grad(loss)(params, batch)
    mahala_grads = grad(mahalanobis_dist)(params, batch, const_flat_params)
    return [(w-step_size*dw - wandb.config.reg_step*jnp.multiply(jnp.abs(dw_mah),jnp.sign(w)),\
             b-step_size*db - wandb.config.reg_step*jnp.multiply(jnp.abs(db_mah),jnp.sign(b)))\
             for (w, b),(dw, db),(dw_mah, db_mah) in zip(params,grads,mahala_grads)]

  # Function which updates model parameters with full regularizer
  @jit
  def update_vol(params, batch, epoch, init_Q):
    grads = grad(loss)(params, batch)
    volume_grads = grad(param_volume)(params, batch, init_Q)
    return [(w - step_size*dw - wandb.config.vol_step*dw_vol, b - step_size*db - wandb.config.vol_step*db_vol)\
           for (w, b),(dw, db),(dw_vol, db_vol) in zip(params,grads,volume_grads)]

  # Function which updates model parameters with L2 regularization
  @jit
  def update_l2(params, batch, epoch):#, init_Q):
    grads = grad(loss)(params, batch)
    reg_grads = grad(l2_regularizer)(params)
    return [(w - step_size*dw - wandb.config.l2_step*dw_reg, b - step_size*db - wandb.config.l2_step*db_reg) \
           for (w, b),(dw, db),(dw_reg, db_reg) in zip(params,grads,reg_grads)]

  @jit
  def update_mahal_vol_jax_bit(params, batch):
    const_flat_params, unflattener = ravel_pytree(params)
    grads = grad(loss)(params, batch)
    mahala_grads = grad(mahalanobis_dist)(params, batch, const_flat_params)
    return grads, mahala_grads

  # Function which updates model parameters with full regularizer
  #@jit
  def update_mahal_vol(params, batch, epoch, init_Q):
    grads, mahala_grads = update_mahal_vol_jax_bit(params, batch)
    volume_grads = grad(param_volume)(params, batch, init_Q)
    return [(w - step_size*dw - wandb.config.reg_step*dw_mah - wandb.config.vol_step*dw_vol, \
             b - step_size*db - wandb.config.reg_step*db_mah - wandb.config.vol_step*db_vol) \
           for (w, b),(dw, db),(dw_mah, db_mah),(dw_vol, db_vol) in zip(params,grads,mahala_grads,volume_grads)]

  # Function which updates model parameters with mahalanobis dist regularizer
  @jit
  def update_mahal_l2(params, batch, epoch):#, init_Q):
    const_flat_params, unflattener = ravel_pytree(params)
    grads = grad(loss)(params, batch)
    mahala_grads = grad(mahalanobis_dist)(params, batch, const_flat_params)
    reg_grads = grad(l2_regularizer)(params)
    return [(w-step_size*dw - wandb.config.l2_step*dw_reg - wandb.config.reg_step*jnp.multiply(jnp.abs(dw_mah),jnp.sign(w)),\
             b-step_size*db - wandb.config.l2_step*db_reg - wandb.config.reg_step*jnp.multiply(jnp.abs(db_mah),jnp.sign(b)))\
             for (w, b),(dw, db),(dw_mah, db_mah),(dw_reg, db_reg) in zip(params,grads,mahala_grads,reg_grads)]

  @jit
  def update_Selective(params, batch, epoch):
    const_flat_params, unflattener = ravel_pytree(params)
    important_params = grad(mahalanobis_dist)(params, batch, const_flat_params)
    grads = grad(loss)(params, batch)
    reg_grads = grad(l2_regularizer)(params)
    return [(w-step_size*dw-wandb.config.selective_l2_step*jnp.where(jnp.abs(dw_choose)>0.001*jnp.max(dw_choose),0.0,dw_reg),\
             b-step_size*db-wandb.config.selective_l2_step*jnp.where(jnp.abs(db_choose)>0.001*jnp.max(db_choose),0.0,db_reg))\
             for (w, b),(dw, db),(dw_reg, db_reg),(dw_choose, db_choose) in zip(params,grads,reg_grads,important_params)]

  # Function which updates model parameters with L2 regularization
  #@jit
  def update_l2_stop(params, batch, epoch):
    if epoch < int(0.7*wandb.config.num_epochs):
        return update_l2(params, batch, epoch)
    else:
        return update_standard(params, batch, epoch)

  #@jit
  def update_l2_stop_selective(params, batch, epoch):
    if epoch < int(0.7*wandb.config.num_epochs):
        return update_l2(params, batch, epoch)
    else:
        return update_Selective(params, batch, epoch)

  # Set hyper-parameters
  wandb.config.layer_sizes = [784, 1024, 1024, 10]
  wandb.config.param_scale_regime_1 = 0.01
  wandb.config.param_scale_regime_2 = 0.1
  wandb.config.param_scale_regime_3 = 2.0
  wandb.config.step_size = 0.001
  wandb.config.reg_step = 0.0001
  wandb.config.l2_step = 0.00001
  wandb.config.selective_l2_step = 0.001
  wandb.config.num_epochs = 400
  wandb.config.batch_size = 128
  wandb.config.num_eigs_keep = 10
  wandb.config.num_trainings = 10
  wandb.config.training_names = ['SGD Small Regime','SGD Large Regime','Metric Small Regime','Metric Large Regime',\
                                 'L2 Large Regime','Selective L2 Large Regime','L2 Stopped Large Regime',\
                                 'Selective L2 Stopped Large Regime']
  wandb.config.updater_index = [0, 0, 1, 1, 3, 6, 7, 8]
  wandb.config.init_index = [1, 2, 1, 2, 2, 2, 2, 2]
  wandb.config.init_type = [wandb.config.param_scale_regime_1, wandb.config.param_scale_regime_2, wandb.config.param_scale_regime_3]
  wandb.config.init_seeds = np.random.choice(100000000, wandb.config.num_trainings, replace=False)

  updaters = [update_standard, update_mahal, update_vol, update_l2,\
              update_mahal_vol, update_mahal_l2,\
              update_Selective, update_l2_stop, update_l2_stop_selective]

  # Loading the dataset
  train_images, train_labels, test_images, test_labels, num_batches, batches = datasets.mnist(wandb.config.batch_size)

  # Create log file and arrays to track the repeated errors
  logs = log_handler.init_logs(len(wandb.config.training_names), wandb.config.num_epochs, wandb.config.num_eigs_keep)

  print("Set of Updates: ", wandb.config.training_names)

  num_params = jnp.sum([i*j + j for i,j in zip(wandb.config.layer_sizes[:-1], wandb.config.layer_sizes[1:])])
  track_Q = np.random.rand(num_params, wandb.config.num_eigs_keep)*100
  for train_index in range(len(wandb.config.training_names)):
      for seed in wandb.config.init_seeds:
          print("############################# New "+wandb.config.training_names[train_index]+" Training ###############################")
          params = init_random_params(wandb.config.init_type[wandb.config.init_index[train_index]], wandb.config.layer_sizes, seed)
          updater = updaters[wandb.config.updater_index[train_index]]
          if wandb.config.init_index[train_index] == 1:        step_size = wandb.config.step_size
          elif wandb.config.init_index[train_index] == 2:      step_size = wandb.config.step_size
          run_logs =  log_handler.init_run_logs() # training, test, time logs, weight norm
          for epoch in range(wandb.config.num_epochs):
              start_time = time.time()
              for _ in range(num_batches):
                params = updater(params, next(batches), epoch)
              epoch_time = time.time() - start_time
              norm = np.sum([np.linalg.norm(param_set[0])+np.linalg.norm(param_set[1]) for param_set in params])

              train_acc = accuracy(params, (train_images, train_labels))
              test_acc = accuracy(params, (test_images, test_labels))
              print("Epoch {} in {:0.2f} sec".format(epoch, epoch_time))
              print("Training set accuracy {}".format(train_acc))
              print("Test set accuracy {}".format(test_acc))
              wandb.log({"Training Type": wandb.config.training_names[train_index], "epoch": epoch, "epoch_time": epoch_time,\
                         wandb.config.training_names[train_index]+" Train Accuracy": float(train_acc),\
                         wandb.config.training_names[train_index]+" Test Accuracy": float(test_acc)})

              run_logs[0] = np.append(run_logs[0], train_acc)
              run_logs[1] = np.append(run_logs[1], test_acc)
              run_logs[2] = np.append(run_logs[2], epoch_time)
              run_logs[3] = np.append(run_logs[3], norm)

          logs[0 + (train_index*5)] = np.vstack([logs[0 + (train_index*5)], run_logs[0].reshape(1,wandb.config.num_epochs)])
          logs[1 + (train_index*5)] = np.vstack([logs[1 + (train_index*5)], run_logs[1].reshape(1,wandb.config.num_epochs)])
          logs[2 + (train_index*5)] = np.vstack([logs[2 + (train_index*5)], run_logs[2].reshape(1,wandb.config.num_epochs)])
          converge_eig_vals, track_Q = param_simultaneous_power_iteration(params, wandb.config.num_eigs_keep,\
                  (train_images, train_labels), track_Q)
          logs[3 + (train_index*5)] = np.vstack([logs[3 + (train_index*5)], converge_eig_vals])
          logs[4 + (train_index*5)] = np.vstack([logs[4 + (train_index*5)], run_logs[3].reshape(1,wandb.config.num_epochs)])

  # Remove place holder first line of zeros used to just define the shape of the arrays to vstack with initially
  all_errors=log_handler.remove_place_holders(logs)

  # Calculate the means of the train and test errors for all types of training, as well as for the difference in errors of the
  # different trainings to the full regularizer (excluding full reg with itself)
  all_means = log_handler.calc_all_means(all_errors, wandb.config.training_names)

  # Calculate the standard deviations of the train and test errors for all types of training, and for the difference in errors of the
  # different trainings to the full regularizer (excluding full reg with itself)
  all_std = log_handler.calc_all_std(all_errors, wandb.config.training_names)

  # Plots the mean training and test error/accuracy (for regression/classification respectively) for each training type using the
  # mean values calculated above with 2*standard deviation error bounds
  log_handler.create_all_plots(all_means, all_std, wandb.config.training_names, "Accuracy", "MNIST Classification", False)

max_stars_repo_name: JarvisDevon/Masters

max_issues_repo_name: JarvisDevon/Masters

max_forks_repo_name: JarvisDevon/Masters

reason: import numpy,import jax,from jax


text:
import os
import json
import tempfile
import numpy as np
from os.path import join as pjoin

from agent import Agent
import evaluate
import generic
import reinforcement_learning_dataset

def run_eval():

    config = generic.load_config()
    agent = Agent(config)
    output_dir = "."
    data_dir = "."

    # make game environments
    requested_infos = agent.select_additional_infos()
    games_dir = "./"

    eval_env, num_eval_game = reinforcement_learning_dataset.get_evaluation_game_env(games_dir + config['rl']['data_path'],
                                                                                config['rl']['difficulty_level'],
                                                                                requested_infos,
                                                                                agent.eval_max_nb_steps_per_episode,
                                                                                agent.eval_batch_size,
                                                                                valid_or_test="test")

    json_file_name = agent.experiment_tag.replace(" ", "_")
    # load pretrained models
    agent.load_pretrained_model(agent.load_from_tag + ".pt", load_partial_graph=False)

    # evaluate
    if agent.real_valued_graph:
        agent.load_pretrained_graph_generation_model(data_dir + "/" + agent.load_graph_generation_model_from_tag + ".pt")
        eval_game_points, eval_game_points_normalized, eval_game_step, detailed_scores = evaluate.evaluate_rl_with_real_graphs(eval_env, agent, num_eval_game)
        command_generation_f1 = 0.0
    else:
        if agent.eval_g_belief:
            agent.load_pretrained_graph_generation_model(data_dir + "/" + agent.load_graph_generation_model_from_tag + ".pt")
            eval_game_points, eval_game_points_normalized, eval_game_step, command_generation_f1, detailed_scores = evaluate.evaluate_belief_mode(eval_env, agent, num_eval_game)
        else:
            eval_game_points, eval_game_points_normalized, eval_game_step, _, detailed_scores = evaluate.evaluate(eval_env, agent, num_eval_game)
            command_generation_f1 = 0.0

    # write accuracies down into file
    _s = json.dumps({"eval game points": str(eval_game_points),
                     "eval normalized game points": str(eval_game_points_normalized),
                     "eval steps": str(eval_game_step),
                     "command generation f1": str(command_generation_f1),
                     "detailed scores": detailed_scores})
    with open(output_dir + "/" + json_file_name + '.json', 'a+') as outfile:
        outfile.write(_s + '\n')
        outfile.flush()

if __name__ == '__main__':
    run_eval()


text:
import os.path
import numpy.testing as npt
import numpy as np
import pandas as pd
import pandas.util.testing as pdt
import pandana as pdna
import pytest

from .. import buffer
from activitysim.core import tracing

@pytest.fixture(scope='module')
def data_dir():
    return os.path.join(os.path.dirname(__file__), 'data')

@pytest.fixture(scope='module')
def configs_dir():
    return os.path.join(os.path.dirname(__file__), 'configs')

@pytest.fixture(scope='module')
def spec_name(data_dir):
    return os.path.join(data_dir, 'buffer_spec.csv')

@pytest.fixture(scope='module')
def net_name(data_dir):
    return os.path.join(data_dir, 'test_net.h5')

@pytest.fixture(scope='module')
def zone_name(data_dir):
    return os.path.join(data_dir, 'test_zones.csv')

@pytest.fixture(scope='module')
def data_name(data_dir):
    return os.path.join(data_dir, 'data.csv')

@pytest.fixture(scope='module')
def data(data_name):
    return pd.read_csv(data_name)

def test_read_model_spec(spec_name):

    spec = buffer.read_buffer_spec(spec_name)

    # assert len(spec) == 8

    assert list(spec.columns) == ['description', 'target', 'variable', 'target_df', 'expression']

def test_buffer_variables(capsys, spec_name, net_name, zone_name):

    spec = buffer.read_buffer_spec(spec_name)

    network = pdna.Network.from_hdf5(net_name)
    zone_data_df = pd.read_csv(zone_name, index_col='zoneid')
    assert len(zone_data_df) == 3

    zone_data_df['node_id'] = network.get_node_ids(zone_data_df['xcoord_p'],
                                                   zone_data_df['ycoord_p'])

    print(zone_data_df.index)
    # zone 1219983 should be assigned to network node 82030 becuase it is the nearest node:
    assert int(zone_data_df.loc[735313].node_id) == 84076
    print(zone_data_df.loc[735313].node_id)

    locals_d = {
        'network': network,
        'zones_df': zone_data_df,
        'node_id': 'node_id'
    }
    # zone_data_df.set_index(constants['zone_index'], inplace=True)
    # zone_data_df.reset_index(level=0, inplace=True)

    # locals_d = {'CONSTANT': 7, '_shadow': 99}

    results, trace_results, trace_assigned_locals \
        = buffer.buffer_variables(spec, 'zones_df', locals_d, trace_rows=None)

    print(results.columns)

    assert list(results.columns) == ['target1', 'target2', 'target3']
    assert list(results.target1) == [382, 382, 382]
    assert list(results.target2) == [1, 1, 1]
    assert list(results.target3) == [383, 383, 383]
    assert trace_results is None
    assert trace_assigned_locals is None

    trace_zone_rows = zone_data_df.index.isin([735313])

    results, trace_results, trace_assigned_locals \
        = buffer.buffer_variables(spec, 'zones_df', locals_d, trace_rows=trace_zone_rows)

    # should get same results as before
    assert list(results.target3) == [383, 383, 383]

    # should assign trace_results for second row in data
    print(trace_results)

    # assert trace_results is not None
    # assert '_temp' in trace_results.columns
    # assert list(trace_results['_scalar']) == [42]

    # shadow should have been assigned
    # assert list(trace_results['_shadow']) == [1]
    # assert list(trace_results['_temp']) == [9]
    # assert list(trace_results['target3']) == [530]

    # print "trace_assigned_locals", trace_assigned_locals
    # assert trace_assigned_locals['_DF_COL_NAME'] == 'thing2'

    # shouldn't have been changed even though it was a target
    # assert locals_d['_shadow'] == 99

    out, err = capsys.readouterr()

max_stars_repo_name: bstabler/netbuffer

max_issues_repo_name: bstabler/netbuffer

max_forks_repo_name: bstabler/netbuffer


text:
from bs4 import BeautifulSoup
from fake_useragent import UserAgent
import requests
import pandas as pd
import numpy as np
import concurrent.futures

class Scraper:
    """
    A class that scrapes ebay for phones based on provided brand, condition and number of items

    ...

    Attributes
    ----------
    brand : string
        name of the brand that you want scraped, example: 'Apple'

    number_of_items : int
        number of phones to be scraped

    condition_name : string
        condition of the phones to be scraped, 'used' or 'new'

    Class variables
    ---------------
    user_agent : UserAgent() object
        UserAgent() object that will be used in the methods

    Methods
    -------
    get_condition():
        returns a code for used or new condition to be used in the url

    get_brand_id():
        returns a code for a given brand to be used in the url

    get_num_of_pages():
        returns number of pages to be used in the url

    get_phones_url():
        returns urls from the main page of ebay and stores the specific model url in a list

    get_single_phone():
        scrapes the single model phone from get_phones_url list, returns a list of features

    scrape_phones():
        main method of the class, creates a csv file in the main directory
    """

    user_agent = UserAgent()

    def __init__(self, brand: str, number_of_items: int, condition_name: str):
        """
        Constructs a default state of attributes.

        Args:
            brand: (string)
                Name of the brand that will be scraped
            number_of_items: (int)
                Number of items that will be scraped
            condition_name: (str)
                Condition type that will be scraped
        """
        self.__brand = brand
        self.__number_of_items = number_of_items
        self.__condition_name = condition_name

    @property
    def condition_code(self) -> int:
        """Function is checking condition_name state in order to assign a correct value code for the url to be scraped

        Raises:
            KeyError: if condition is not 'used' or 'new'

        Returns:
            int: specific code for a given condition
        """
        condition_code = {"new": 1000, "used": 3000}

        try:
            return condition_code[self.__condition_name]
        except KeyError:
            raise KeyError("condition_name must be a string 'new' or 'used'")

    @property
    def brand_id(self) -> int:
        """Function is checking brand state in order to assign a correct value code for the url to be scraped

        Raises:
            KeyError: if brand is not 'Apple', 'LG', 'Huawei', 'Samsung'

        Returns:
            int: specific code for a given brand
        """
        brand_id_codes = {
            "Apple": 319682,
            "LG": 353985,
            "Huawei": 349965,
            "Samsung": 352130}

        try:
            return brand_id_codes[self.__brand]
        except KeyError:
            raise KeyError("brand must be 'Apple', 'LG', 'Huawei' or 'Samsung'")

    @property
    def num_of_pages(self) -> int:
        """Calculates number of pages that will be scraped based on number of items user wants to get.
            By default, each page has 48 items.

        Returns:
            int: number of pages that will be scraped
        """
        try:
            return int(round(self.__number_of_items / 48))
        except TypeError:
            raise TypeError("number_of_items must be of int type")

    def get_phones_url(self) -> list:
        """Extracts url from each item on the main page of ebay, returns a list of urls.

        Returns:
            list: list of single phones
        """
        urls = []

        for page_number in range(self.number_of_pages):
            url = f"https://www.ebay.com/b/{self.__brand}-Cell-Phones-Smartphones/9355/bn_{self.brand_id}?LH_ItemCondition={self.condition_code}&LH_PrefLoc=5&LH_Sold=1&rt=nc&_pgn={page_number}"
            page = requests.get(url, headers={"User-Agent": self.user_agent.google})
            soup = BeautifulSoup(page.content, "html.parser")

            urls.extend([links.get('href', np.nan) for links in soup.find_all('a', class_="s-item__link")])

        return urls

    def get_single_phone(self, url: str) -> list:
        """ It accepts a url, scrapes the url, assigns scraped data to a list, returns a list

        Args:
            url (str): url to be scraped

        Returns:
            list: list of scraped items
        """
        page = requests.get(url, headers={"User-Agent": self.user_agent.google})
        soup = BeautifulSoup(page.content, "html.parser")

        temp_price = np.nan
        temp_model = np.nan
        temp_ram = np.nan
        temp_storage = np.nan
        temp_processor = np.nan
        temp_camera = np.nan

        if soup.find(class_="display-price"):
            temp_price = soup.find(class_="display-price").get_text()

        for item in soup.find_all('div', class_='s-name'):
            if item.get_text() == "Model": temp_model = item.next_sibling.get_text()
            if item.get_text() == "RAM": temp_ram = item.next_sibling.get_text()
            if item.get_text() == "Storage Capacity": temp_storage = item.next_sibling.get_text()
            if item.get_text() == "Processor": temp_processor = item.next_sibling.get_text()
            if item.get_text() == "Camera Resolution": temp_camera = item.next_sibling.get_text()

        return [temp_price, temp_model, temp_ram, temp_storage, temp_processor, temp_camera]

    def scrape_phones(self) -> None:
        """Main function of the package. Scrapes the data with other functions, creates a dataframe,
        exports the dataframe as csv file.

        Returns:
            csv_file: exports csv file in the main directory with scraped data
        """

        phone_model, phone_ram, phone_storage, phone_processor, phone_camera, phone_price = ([] for i in range(6))

        phones_url = self.get_phones_url()
        fetch_single_phone = self.get_single_phone()

        with concurrent.futures.ThreadPoolExecutor() as executor:
            results = executor.map(fetch_single_phone, phones_url)

            for result in results:
                phone_price.append(result[0])
                phone_model.append(result[1])
                phone_ram.append(result[2])
                phone_storage.append(result[3])
                phone_processor.append(result[4])
                phone_camera.append(result[5])

        df = pd.DataFrame(
            data=zip(
                phone_price,
                phone_model,
                phone_ram,
                phone_storage,
                phone_processor,
                phone_camera),
            columns=['price', 'model', 'ram', 'storage', 'processor', 'camera']
            )

        df['brand'] = self.__brand
        df['condition'] = self.__condition_name

        return df.to_csv(f"{self.__brand}_{self.__condition_name}_data.csv", index=False)

max_stars_repo_name: LeonMilosevic/ebay-phone-scraper

max_issues_repo_name: LeonMilosevic/ebay-phone-scraper

max_forks_repo_name: LeonMilosevic/ebay-phone-scraper


text:
import pandas as pd
import numpy as np
import requests
from bs4 import BeautifulSoup

from .exceptions import BadRequestError

def get_stations():

    url = 'https://coagmet.colostate.edu/station_index.php'
    response = requests.get(url)
    soup = BeautifulSoup(response.text, features='lxml')
    table = soup.find('table', attrs={'class': 'station-index'})

    output_rows = []
    for table_row in table.findAll('tr'):
        columns = table_row.findAll('td')
        output_row = []
        for column in columns:
            output_row.append(column.text)
        output_rows.append(output_row)

    values = []
    station_ids = []
    for i, row in enumerate(output_rows):
        if len(row) > 0:
            station_ids.append(row[0])
            row_vals = []
            for val in row[1:]:
                try:
                    val = float(val)
                except ValueError:
                    pass
                row_vals.append(val)
            values.append(row_vals)

    values = np.array(values)

    headers = [header.text for header in table.findAll('th')]
    headers = headers[:len(headers)]
    headers.remove('ID')

    dict_ = {headers[i]: values[:, i] for i in range(values.shape[1])}

    df = pd.DataFrame(dict_, index=station_ids)

    if len(df) == 1 and df[df.columns[1:]].isnull().values.all():
        raise BadRequestError(response.text)

    df['Latitude'] = df['Latitude'].astype(float)
    df['Longitude'] = df['Longitude'].astype(float)
    df['Elev. (ft)'] = df['Elev. (ft)'].astype(float)

    for i, row in df.iterrows():
        for col in ['First Obs.', 'Last Obs.']:
            try:
                df.at[i, col] = pd.to_datetime(row[col],
                                                format='%b %d, %Y')
            except ValueError:
                break

    return df

class Stations:

    def __init__(self):
        self.df = None

    def get(self):
        self.df = get_stations()
        return self.df

    def find_nearest_stations(self, lat, lon, n=3):
        if self.df is None:
            self.get()
        location = np.array([lat, lon])
        locations = np.array(list(zip(self.df['Latitude'],
                                      self.df['Longitude'])))
        dists = np.sum((locations - location) ** 2, axis=1)
        return self.df.index[np.argsort(dists)[:n]]

    def find_nearest_station(self, lat, lon):
        if self.df is None:
            self.get()
        return self.find_nearest_stations(lat, lon, n=1)[0]

    def get_station(self, station_id):
        if self.df is None:
            self.get()
        if isinstance(station_id, tuple):
            lon, lat = station_id[0], station_id[1]
            station_id = self.find_nearest_station(lon, lat)
        if not isinstance(station_id, str):
            raise ValueError('`station_id` must be either <str> or <tuple>,'
                            ' got: {}'.format(station_id))
        return self.df.loc[station_id]

__all__ = ['Stations']

max_stars_repo_name: marcodlk/coagmet

max_issues_repo_name: marcodlk/coagmet

max_forks_repo_name: marcodlk/coagmet