text:
import numpy as np
from sla import SLA
from app import App
import random
from itertools import product
from ortools.linear_solver import pywraplp

class Market:
    def __init__(self, apps):
        self.apps = apps

        self.period_length =  10 # length of period (minutes)

        ### CORE PARAMETERS
        # This market contains only Xeon cores with 2 power modes,
        # using data from Guevara.
        # mode 0 = sleep, mode 1 = active
        self.G = 1 # number machine groups
        self.M = 2 # number power modes
        self.MACHINES = [15] # number total machines in each group
        self.CORES = [4] # number cores per machine in each group
        self.cycles_per_transaction = 300 # MCycles per transaction

        # cycles supplied by cores (MCycles / minute)
        self.gamma = lambda g, t : [
            [0, 2.5]
        ][g][t] * 1000 * 60
        # time to switch power modes (minutes)
        self.delta = lambda g, f, t : [
            [0, 8],
            [6, 0]
        ][f][t] / 60

        self.E_mult = 1.6
        self.E_cost = .07 / (1000 * 60) # cost per unit energy ($ / (W * min))
        self.E_base_active = lambda g, tau, t : [25, 65][t] * tau # energy used for platform (W * min)
        self.E_base_idle = lambda g, tau, t : [25, 65][t] * tau # energy used for unsold platforms (W * min)
        self.E_trans = lambda g, f, t : [ # energy used for transition for each machine (W * min)
            [0, 65],
            [65, 0]
        ][f][t] * self.delta(g, f, t)
        self.E_core_active = lambda g, tau, t : [ # marginal energy for core (W * min)
            [0, 15.6]
        ][g][t] * tau
        self.E_core_idle = lambda g, tau, t : [ # marginal energy for unsold cores (W * min)
            [0, 7.8]
        ][g][t] * tau
        self.H_trans = lambda g, f, t : 0 if f == t else .05 # cost of transitioning ($)
        self.H_base = lambda g, tau, t : 0 if t == 0 else .0001 * tau # cost of maintainting machine ($), chosen arbitrarily

        self.current_time = 0 # start time of the next allocation period

        # allocation from the last completed period
        self.current_core_allocation = { app_id : { (g,f) : 0 for g, f in product(range(self.G), range(self.M)) } for app_id in self.apps }
        self.current_unsold_cores = { (g,f) : (0 if f != 0 else self.CORES[g] * self.MACHINES[g]) for g, f in product(range(self.G), range(self.M)) }

        # temp variables for storage
        self.saved_core_allocation = None
        self.saved_unsold_cores = None

    '''
    Runs the allocation for the period starting at current_time.
    Effects:
        - Updates current_core_allocation, current_unsold_cores, current_time, and each app's current price/cycles
        - Returns welfare of this period's allocation, user values, VCG prices
    Inputs:
        - use_VCG_price : bool or list; Indicates which apps should pay the VCG price (True = all, False = none)
        - allow_holding : bool; Indicates whether apps can choose to hold their cores from last period (done greedily by the app)
    '''
    def get_allocation_and_prices(self, use_VCG_price=False, allow_holding=False):
        # see if each user will hold their last period allocation
        holding=[]
        for app_id, app in self.apps.items():
            if allow_holding and app.will_hold(self.current_time, self.period_length):
                holding.append(app_id)

        # realize the demands for the current time
        for app in self.apps.values():
            app.set_demand(self.current_time)

        # run the actual allocation and calculate VCG pricing
        VCG = {}
        total_welfare, total_values, total_cycles = self.allocate_all_apps(holding)
        for app_id in self.apps:
            welfare, _, _ = self.allocate_without_app(app_id, holding)
            price = welfare - (total_welfare - total_values[app_id])
            VCG[app_id] = price

        # store price and allocated cycles in each app
        for app_id in self.apps:
            if app_id in holding: # don't change price or alloc if holding
                continue
            if (type(use_VCG_price) == bool and use_VCG_price) or (type(use_VCG_price) != bool and app_id in use_VCG_price):
                self.apps[app_id].current_price = VCG[app_id]
            else:
                self.apps[app_id].current_price = total_values[app_id]
            self.apps[app_id].current_cycles = total_cycles[app_id]

        # finalize current allocation in the datacenter
        self.current_core_allocation = self.saved_core_allocation
        self.current_unsold_cores = self.saved_unsold_cores
        self.saved_core_allocation = None
        self.saved_unsold_cores = None

        # advance the time to next period
        self.current_time += self.period_length

        return total_welfare, total_values, VCG

    def allocate_all_apps(self, holding=[]):
        return self.market_allocate(True, [], holding)

    def allocate_without_app(self, id_to_exclude, holding=[]):
        return self.market_allocate(False, [id_to_exclude], holding)

    def market_allocate(self, save_alloc=False, exclude_from_obj=[], holding=[]):

        solver = pywraplp.Solver.CreateSolver('SCIP')
        infinity = solver.infinity()

        C_current = {}
        for g, f in product(range(self.G), range(self.M)):
            C_current[(g, f)] = sum([self.current_core_allocation[app_id][(g, f)] for app_id in self.apps]) + self.current_unsold_cores[(g, f)]

        #############################################
        # Define MIP
        #############################################

        # buyer valuation model
        V = {} # value provided to application a V = F(Q) ($)
        Q = {} # cycles provided to application a (MCycles)
        Q_seq = {} # sequential cycles provided to application a
        Q_par = {} # parallel cycles provided to application a
        Z_seg = {} # segment indicator for piecewise linear function
        Q_seg = {} # segment value for piecewise linear function
        C_sold = {}
        C_sold_seq = {} # sold cores used sequentially
        C_unsold = {}
        C_partunsold = {}
        for a, A in self.apps.items():
            V[a] = solver.NumVar(0, infinity, f'V[{a}]')
            Q[a] = solver.NumVar(0, infinity, f'Q[{a}]')
            Q_seq[a] = solver.NumVar(0, infinity, f'Q_seq[{a}]')
            Q_par[a] = solver.NumVar(0, infinity, f'Q_par[{a}]')

            # set measured application demand for A
            A.sla.compute_supply_value(A.current_demand, self.period_length)

            # auxiliary for computing V = F(Q)
            Z_seg[a] = {}
            Q_seg[a] = {}
            x_prev = lambda s : 0 if s == 0 else A.sla.supply_x[s-1] * self.cycles_per_transaction * self.period_length
            x_curr = lambda s : A.sla.supply_x[s] * self.cycles_per_transaction * self.period_length
            y_prev = lambda s : 0 if s == 0 else A.sla.value_y[s-1]
            y_curr = lambda s : A.sla.value_y[s]
            # iterate over segments
            num_seg = len(A.sla.supply_x)
            for s in range(num_seg):
                Z_seg[a][s] = solver.IntVar(0, 1, f'Z_seg[{a}][{s}]')
                Q_seg[a][s] = solver.NumVar(0, infinity, f'Q_seg[{a}][{s}]')
                # ^Q_{s-1} Z_s <= Q_s <= ^Q_{s} Z_s
                solver.Add(x_prev(s) * Z_seg[a][s] <= Q_seg[a][s])
                solver.Add(x_curr(s) * Z_seg[a][s] >= Q_seg[a][s])
            # Z_seg[a][s] sum to 1
            solver.Add(sum(Z_seg[a][s] for s in range(num_seg)) == 1)
            # Q_seg[a][s] sum to Q[a]
            solver.Add(sum(Q_seg[a][s] for s in range(num_seg)) == Q[a])
            # set V = F(Q) for piecewise linear F
            solver.Add(sum(
                y_prev(s) * Z_seg[a][s] +
                (y_curr(s) - y_prev(s)) / (x_curr(s) - x_prev(s)) *
                (Q_seg[a][s] - x_prev(s) * Z_seg[a][s])
                for s in range(num_seg)
            ) == V[a])

            # set Q
            C_sold[a] = {}
            C_sold_seq[a] = {}
            for g, f, t in product(range(self.G), range(self.M), range(self.M)):
                C_sold[a][(g,f,t)] = solver.IntVar(0, infinity, f'C_sold[{a}][{(g,f,t)}]')
                C_sold_seq[a][(g,f,t)] = solver.IntVar(0, 1, f'C_sold_seq[{a}][{(g,f,t)}]')

                # need to have bought sequential core
                solver.Add(C_sold_seq[a][(g,f,t)] <= C_sold[a][(g,f,t)])
            #solver.Add(C_sold[0][(0,0,1)] == 4)

            # each application can use at most one core sequentially
            solver.Add(sum(
                C_sold_seq[a][(g,f,t)]
                for g, f, t in product(range(self.G), range(self.M), range(self.M))
            ) <= 1)

            # set number of sequential cycles provided
            solver.Add(sum(
                self.gamma(g, t) * (self.period_length - self.delta(g, f, t)) * C_sold_seq[a][(g,f,t)]
                for g, f, t in product(range(self.G), range(self.M), range(self.M))
            ) == Q_seq[a])

            # set number of parallel cycles provided
            solver.Add(sum(
                self.gamma(g, t) * (self.period_length - self.delta(g, f, t)) * C_sold[a][(g,f,t)]
                for g, f, t in product(range(self.G), range(self.M), range(self.M))
            ) == Q_par[a])

            # set number of cycles provided
            solver.Add(
                (1 - A.parallel_fraction) * Q_seq[a] + A.parallel_fraction * Q_par[a]
            == Q[a])

        # goods in the market
        M_sold = {}
        M_unsold = {}
        for g, f, t in product(range(self.G), range(self.M), range(self.M)):
            M_sold[(g,f,t)] = solver.IntVar(0, infinity, f'M_sold[{(g,f,t)}]')
            M_unsold[(g,f,t)] = solver.IntVar(0, infinity, f'M_unsold[{(g,f,t)}]')
            C_unsold[(g,f,t)] = solver.IntVar(0, infinity, f'C_unsold[{(g,f,t)}]')
        for g, t in product(range(self.G), range(self.M)):
            C_partunsold[(g,t)] = solver.IntVar(0, infinity, f'C_partunsold[{(g,t)}]')

        for g, t in product(range(self.G), range(self.M)):
            solver.Add(
                sum(self.CORES[g] * M_sold[(g,f,t)] for f in range(self.M)) ==
                sum(
                    C_sold[a][(g,f,t)]
                    for a, f in product(self.apps.keys(), range(self.M))
                ) + C_partunsold[(g,t)]
            )
            solver.Add(
                sum(self.CORES[g] * M_unsold[(g,f,t)] for f in range(self.M)) ==
                sum(C_unsold[(g,f,t)] for f in range(self.M)) - C_partunsold[(g,t)]
            )
        for g in range(self.G):
            solver.Add(
                self.MACHINES[g] == sum(
                    M_sold[(g,f,t)] + M_unsold[(g,f,t)]
                    for f, t in product(range(self.M), range(self.M))
                )
            )
        for g, f in product(range(self.G), range(self.M)): # maintain consistency with current core states
            solver.Add( sum( C_sold[a][(g,f,t)] for a, t in product(self.apps.keys(), range(self.M)))
                    + sum( C_unsold[(g,f,t)] for t in range(self.M) )
                    == C_current[(g, f)] )

        # keep allocation of those holding
        for a in holding:
            for g, f, t in product(range(self.G), range(self.M), range(self.M)):
                if f == t:
                    solver.Add( C_sold[a][(g,f,f)] == self.current_core_allocation[a][(g,f)] )
                else:
                    solver.Add( C_sold[a][(g,f,t)] == 0 )

        # seller cost model
        E = solver.NumVar(0, infinity, 'E')
        H = solver.NumVar(0, infinity, 'H')
        solver.Add(
            E == sum(
                self.E_mult * (self.E_trans(g,f,t) + self.E_base_active(g, self.period_length, t)) *
                M_sold[(g,f,t)]
                for g, f, t in product(range(self.G), range(self.M), range(self.M))
            ) + sum(
                self.E_mult * self.E_core_active(g, self.period_length, t) * C_sold[a][(g,f,t)]
                for a, g, f, t in product(self.apps.keys(), range(self.G), range(self.M), range(self.M))
            ) + sum(
                self.E_mult * (self.E_trans(g,f,t) + self.E_base_idle(g, self.period_length, t)) *
                M_unsold[(g,f,t)]
                for g, f, t in product(range(self.G), range(self.M), range(self.M))
            ) + sum(
                self.E_mult * self.E_core_idle(g, self.period_length, t) * C_unsold[(g,f,t)]
                for g,f,t in product(range(self.G), range(self.M), range(self.M))
            )
        )
        solver.Add(
            H == sum(
                (self.H_trans(g,f,t) + self.H_base(g, self.period_length, t)) * (M_sold[(g,f,t)] + M_unsold[(g,f,t)])
                for g, f, t in product(range(self.G), range(self.M), range(self.M))
            )
        )

        #print('Number of variables =', solver.NumVariables())
        #print('Number of constraints =', solver.NumConstraints())

        # objective ($)
        solver.Maximize(
            sum(V[a] for a in self.apps.keys() if a not in exclude_from_obj) - (self.E_cost * E) - H
        )

        status = solver.Solve()

        #print(self.apps[0].sla.eval_value(Q[0].solution_value() / (self.period_length * self.cycles_per_transaction), self.apps[0].current_demand))

        if status == pywraplp.Solver.OPTIMAL:
            #print('Solution:')
            #print('Objective value =', solver.Objective().Value())
            if save_alloc:
                self.saved_core_allocation = { a : {} for a in self.apps.keys() }
                self.saved_unsold_cores = {}
                for a, g, t in product(self.apps.keys(), range(self.G), range(self.M)):
                    self.saved_core_allocation[a][(g,t)] = sum([C_sold[a][(g,f,t)].solution_value() for f in range(self.M)])
                    self.saved_unsold_cores[(g,t)] = sum([C_unsold[(g,f,t)].solution_value() for f in range(self.M)])
            return solver.Objective().Value(), {a : V[a].solution_value() for a in V}, {a : [Q[a].solution_value(), Q_seq[a].solution_value(), Q_par[a].solution_value()] for a in Q}

        else:
            print('The problem does not have an optimal solution.')

max_stars_repo_name: Cephian/marketecture

max_issues_repo_name: Cephian/marketecture

max_forks_repo_name: Cephian/marketecture


text:
"""
Copyright (c) 2018-2020 Intel Corporation

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import numpy as np

from .adapter import Adapter
from ..representation import QualityAssessmentPrediction

class QualityAssessmentAdapter(Adapter):
    __provider__ = 'face_recognition_quality_assessment'
    predcition_types = (QualityAssessmentPrediction, )
    score_weight = [0.25, 0.50, 0.50, 0.75]

    def process(self, raw, identifiers=None, frame_meta=None):
        prediction = self._extract_predictions(raw, frame_meta)[self.output_blob]
        return [QualityAssessmentPrediction(identifier, quality_assessment=self.calculate_quality(embedding))
                for identifier, embedding in zip(identifiers, prediction)]

    def calculate_quality(self, qa_feature):
        # Sort by feature value and get top1 and top2 id
        softed_feature_idx = np.argsort(qa_feature)[::-1]

        top_id1 = softed_feature_idx[0]
        top_score1 = qa_feature[top_id1]

        top_id2 = softed_feature_idx[1]
        top_score2 = qa_feature[top_id2]

        # Calculate quality score
        qs_val = 0.0
        if top_score1 > (top_score2 * 2):
            if top_id1 <= 1:
                qs_val = self.score_weight[top_id1] - 0.25 * top_score1
            else:
                qs_val = self.score_weight[top_id1] + 0.25 * top_score1
        else:
            score_sum = top_score1 + top_score2

            # Get top1 val
            top1_w = top_score1 / score_sum
            max_val1 = 0.0
            qs_val1 = 0.0

            if top_id1 <= 1:
                max_val1 = self.score_weight[top_id1]
                qs_val1 = max_val1 - 0.25 * top1_w
            else:
                qs_val1 = self.score_weight[top_id1] + 0.25 * top1_w
                max_val1 = self.score_weight[top_id1] + 0.25

            # Get top2 val
            top2_w = top_score2 / score_sum
            max_val2 = 0.0
            qs_val2 = 0.0

            if top_id2 <= 1:
                max_val2 = self.score_weight[top_id2]
                qs_val2 = max_val2 - 0.25 * top2_w
            else:
                qs_val2 = self.score_weight[top_id2] + 0.25 * top2_w
                max_val2 = self.score_weight[top_id2] + 0.25

            qs_val = (qs_val1 + qs_val2) / (max_val1 + max_val2)

        return round(qs_val, 6)

ext: py

reason: import numpy


text:
"""
C code printer
"""

from str import StrPrinter
from sympy.printing.precedence import precedence, PRECEDENCE
from sympy.core.basic import S

class CCodePrinter(StrPrinter):
    """A printer to convert python expressions to stings of c code"""
    printmethod = "_ccode_"

    def _print_Pow(self, expr):
        PREC = precedence(expr)
        if expr.exp is S.NegativeOne:
            return '1/%s'%(self.parenthesize(expr.base, PREC))
        else:
            return 'pow(%s,%s)'%(self.parenthesize(expr.base, PREC),
                                 self.parenthesize(expr.exp, PREC))

    def _print_Exp1(self, expr):
        return "exp(1)"

    def _print_Piecewise(self, expr):
        ecpairs = ["(%s) {\n%s\n}\n" % (self._print(c), self._print(e)) \
                       for e, c in expr.args[:-1]]
        last_line = ""
        if expr.args[-1].cond is S.One:
            last_line = "else {\n%s\n}" % self._print(expr.args[-1].expr)
        else:
            ecpairs.append("(%s) {\n%s\n" % \
                           (self._print(expr.args[-1].cond),
                            self._print(expr.args[-1].expr)))
        code = "if %s" + last_line
        return code % "else if ".join(ecpairs)

def ccode(expr):
    r"""Converts an expr to a string of c code

        Works for simple expressions using math.h functions.

        >>> from sympy import *
        >>> from sympy.abc import *

        >>> ccode((2*tau)**Rational(7,2))
        '8*pow(2,(1/2))*pow(tau,(7/2))'
    """
    return CCodePrinter().doprint(expr)

def print_ccode(expr):
    """Prints C representation of the given expression."""
    print ccode(expr)

max_stars_repo_name: gnulinooks/sympy

max_issues_repo_name: gnulinooks/sympy

max_forks_repo_name: gnulinooks/sympy

reason: from sympy


text:
import os
import datetime

from astropy.io import fits
import numpy as np

from crds.core import rmap
from crds import data_file

from . import utils

_TIMESTAMP_FORMAT = "%b %d %Y %H:%M:%S"

_AM_PM_TIMESTAMP_FORMAT = "%b %d %Y %I:%M:%S:%f%p"

_PEDIGREE_DATE_FORMAT = "%d/%m/%Y"

_DBTABLE_BY_REFTYPE = {
    utils.THROUGHPUT_LOOKUP_REFTYPE: "CRCOMPLIST",
    utils.THERMAL_LOOKUP_REFTYPE: "CRTHERMLIST",
}

_DESCRIP_BY_REFTYPE = {
    utils.THROUGHPUT_LOOKUP_REFTYPE: "The lookup table for HST component throughputs",
    utils.THERMAL_LOOKUP_REFTYPE: "The lookup table for HST component thermal characteristics",
}

_HISTORY_DESCRIPTION_BY_REFTYPE = {
    utils.THROUGHPUT_LOOKUP_REFTYPE: "throughput",
    utils.THERMAL_LOOKUP_REFTYPE: "thermal",
}

class SynphotLookupGenerator:
    """
    Given a derived-from context and a set of newly delivered synphot files,
    produce an updated lookup file.
    """
    def __init__(self, context):
        self._context = context

    def generate(self, reftype, delivered_files):
        """
        Generate and return an HDUList containing an updated lookup table
        for the specified reftype.
        """
        # Historically ReDCaT has always set this to the current
        # local time and not UTC.
        timestamp = datetime.datetime.today().strftime(_TIMESTAMP_FORMAT)

        original_ref_path = utils.get_cache_path(self._context, reftype, error_on_missing=False)
        if original_ref_path is None:
            return self._generate_hdul(reftype, delivered_files, timestamp)
        else:
            with data_file.fits_open(original_ref_path) as original_ref:
                return self._generate_hdul(reftype, delivered_files, timestamp, original_ref=original_ref)

    def _generate_hdul(self, reftype, delivered_files, timestamp, original_ref=None):
        if original_ref is not None:
            original_table = original_ref[-1].data
            new_time = original_table["TIME"]
            new_compname = original_table["COMPNAME"]
            new_filename = original_table["FILENAME"]
            new_comment = original_table["COMMENT"]
        else:
            new_time = np.chararray((0,), unicode=True)
            new_compname = np.chararray((0,), unicode=True)
            new_filename = np.chararray((0,), unicode=True)
            new_comment = np.chararray((0,), unicode=True)

        updated_instruments = set()
        created_instruments = set()
        for file in delivered_files:
            with data_file.fits_open(file) as hdul:
                component = hdul[0].header["COMPNAME"]
                lookup_filename = utils.get_lookup_filename(component, os.path.basename(file))
                description = hdul[0].header.get("DESCRIP", "")
                if component in new_compname:
                    idx = np.argwhere(new_compname == component)[0][0]
                    new_time[idx] = timestamp
                    new_filename[idx] = lookup_filename
                    new_comment[idx] = description
                    updated_instruments.add(utils.get_instrument(component))
                else:
                    new_time = np.append(new_time, timestamp)
                    new_compname = np.append(new_compname, component)
                    new_filename = np.append(new_filename, lookup_filename)
                    new_comment = np.append(new_comment, description)
                    created_instruments.add(utils.get_instrument(component))

        ind = np.argsort(new_compname)
        columns = [
            fits.Column(name="TIME", format="A26", array=new_time[ind], disp="A26"),
            fits.Column(name="COMPNAME", format="A18", array=new_compname[ind], disp="A18"),
            fits.Column(name="FILENAME", format="A56", array=new_filename[ind], disp="A56"),
            fits.Column(name="COMMENT", format="A68", array=new_comment[ind], disp="A68")
        ]

        new_table_hdu = fits.BinTableHDU.from_columns(columns)

        new_primary_hdu = fits.PrimaryHDU()
        header = new_primary_hdu.header
        header["USEAFTER"] = timestamp
        header["INSTRUME"] = "HST"
        header["COMMENT"] = "Reference file automatically generated by CRDS"
        header["DBTABLE"] = _DBTABLE_BY_REFTYPE[reftype]
        header["DESCRIP"] = _DESCRIP_BY_REFTYPE[reftype]
        header["PEDIGREE"] = self._make_pedigree(new_table_hdu)

        if original_ref:
            for item in original_ref[0].header["HISTORY"]:
                header.add_history(item)

            header.add_history(" ")
            header.add_history("Updated on {}".format(timestamp))
        else:
            header.add_history("Created on {}".format(timestamp))

        header.add_history(" ")
        self._add_instrument_history(reftype, header, updated_instruments, created_instruments)

        return fits.HDUList([new_primary_hdu, new_table_hdu])

    def _add_instrument_history(self, reftype, header, updated_instruments, created_instruments):
        description = _HISTORY_DESCRIPTION_BY_REFTYPE[reftype]

        if len(updated_instruments) > 0:
            instrument_list = self._make_instrument_list(updated_instruments)
            header.add_history("Updated {} tables for {}".format(description, instrument_list))

        if len(created_instruments) > 0:
            instrument_list = self._make_instrument_list(created_instruments)
            header.add_history("Created {} tables for {}".format(description, instrument_list))

    def _make_instrument_list(self, instruments):
        instruments = sorted([i.upper() for i in instruments])
        if len(instruments) == 1:
            return instruments[0]
        elif len(instruments) == 2:
            return instruments[0] and instruments[1]
        else:
            return ", ".join(instruments[:-1]) + ", and " + instruments[-1]

    def _make_pedigree(self, table_hdu):
        times = [self._parse_time(t) for t in table_hdu.data["TIME"]]
        return "INFLIGHT {} {}".format(
            min(times).strftime(_PEDIGREE_DATE_FORMAT),
            max(times).strftime(_PEDIGREE_DATE_FORMAT)
        )

    def _parse_time(self, time):
        time = time.strip().lower()
        if "am" in time or "pm" in time:
            return datetime.datetime.strptime(time, _AM_PM_TIMESTAMP_FORMAT)
        else:
            return datetime.datetime.strptime(time, _TIMESTAMP_FORMAT)

max_stars_repo_name: rendinam/crds

max_issues_repo_name: rendinam/crds

max_forks_repo_name: rendinam/crds

reason: import numpy,from astropy


text:
# Copyright 2017, Xavier Snelgrove
import argparse
import os
import sys
import numpy as np
from scipy import ndimage
import gram
from gram import JoinMode

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Synthesize image from texture", formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    parser.add_argument('--output-width', '-ow', default=512, type=int,
            help="Pixel width of generated image")
    parser.add_argument('--output-height', '-oh', type=int,
            help="Pixel height of generated image. If not specified, equal to output-width.")
    parser.add_argument('--octaves', '-o',  type=int, default=4,
            help="Number of octaves (where 1 means \"Consider only 1 scale\")")
    parser.add_argument('--octave-step', '-os',  type=int, default=1,
            help="Can potentially skip some octaves")
    parser.add_argument('--layers', '-l',  type=int, nargs='+', default=[2, 7],
            help="Which layers to match gram matrices on")
    parser.add_argument('--max-iter', '-m', type=int, default=500,
            help="Maximum iterations for the L-BFGS-B optimizer")
    parser.add_argument("--output-prefix", "-op", default='out',
            help="Prefix to append to output directory")
    parser.add_argument("--save-every", "-n", default=10, type=int,
            help="Save an in-progress optimization image every SAVE_EVERY iterations")
    parser.add_argument("--source-scale", "-ss", type=float,
            help="How much to scale the source image by")
    parser.add_argument("--source-width", "-sw", type=int,
            help="Scale source to this width. Mutually exclusive with source-scale")
    parser.add_argument("--padding-mode", "-p", type=str, choices = ['valid', 'same'], default='valid',
            help="What boundary condition to use for convolutions")
    parser.add_argument("--internal", action="store_true")

    parser.add_argument("--count", "-c", type=int, default=1,
            help="How many images to generate simultaneously")
    parser.add_argument("--mul", type=float, default=1.0, help="Multiply identity target by this amount")
    parser.add_argument("--if-weight", type=float, default=1., help="Inter-frame loss weight")
    parser.add_argument("--if-shift", type=float, default=5., help="How many pixel-shift should inter-frame loss approximate?")
    parser.add_argument("--if-order", type=int, default=2., help="How many frames should we 'tie' together?")
    parser.add_argument("--seed", type=str, choices = ['random', 'symmetric'], default='random', help="How to seed the optimization")
    parser.add_argument("--data-dir", "-d", type=str, default="model_data", help="Where to find the VGG weight files")
    parser.add_argument("--output-dir", type=str, default="outputs", help="Where to save the generated outputs")
    parser.add_argument("--tol", type=float, default=1e-9, help="Gradient scale at which to terminate optimization")

    args = parser.parse_args()

    output_size = (args.output_width, args.output_height if args.output_height is not None else args.output_width)

    if not os.path.exists(args.output_dir):
        os.mkdir(args.output_dir)

    output_dir = "{}.L{}.o{}".format(args.output_prefix, ",".join(str(l) for l in args.layers), args.octaves)
    output_dir = os.path.join(args.output_dir, output_dir)
    if args.internal:
        output_dir += ".internal"
    if args.source_scale:
        output_dir += ".w{:.2}".format(args.source_scale)
    if args.source_width:
        output_dir += ".w{}".format(args.source_width)
    if args.count > 1:
        output_dir += ".c{}".format(args.count)
    if args.mul != 1.0:
        output_dir += ".m{}".format(args.mul)
    if args.octave_step != 1:
        output_dir += ".os{}".format(args.octave_step)

    output_dir += ".{}x{}".format(*output_size)

    suffix = 0
    base_output_dir = output_dir
    while os.path.exists(output_dir):
        output_dir = base_output_dir + ".{}".format(suffix)
        suffix += 1
        if suffix > 100:
            print("Hmm, maybe in an infinite loop trying to create output directory")
            sys.exit(1)

    try:
        os.mkdir(output_dir)
    except OSError:
        print("Hmm, failed to make output directory... race condition?")
        sys.exit(1)

    # Save the command for good measure
    with open(os.path.join(output_dir, "Acommand.txt"), 'w') as f:
        f.write(' '.join(sys.argv))

    width = args.output_width
    height = args.output_height or width

    print("About to generate a {}x{} image, maximizing Gram novelty for layers {} at {} distinct scales".format(width, height, args.layers, args.octaves))

    pyramid_model = gram.make_pyramid_model(args.octaves, args.padding_mode)

    pyramid_gram_model = gram.make_pyramid_gram_model(pyramid_model, args.layers, data_dir=args.data_dir)

    x0 = np.random.randn(args.count, height, width, 3)

    if args.seed == 'symmetric':
        x0 = x0 + x0[:,::-1,    :, :]
        x0 = x0 + x0[:,   :, ::-1, :]
        blur_radius = 50
        for i in range(3):
            x0[...,i] = blur_radius*50*ndimage.gaussian_filter(x0[...,i], blur_radius)
        x0 += np.random.randn(*(x0.shape)) * 2
    else:
        # Shift the whole thing to be near zero
        x0 += 10 - gram.colour_offsets

    gram.synthesize_novelty(pyramid_gram_model,
            width = width, height = height, frame_count=args.count, mul=args.mul,
            octave_step = args.octave_step,
            internal = args.internal,
            x0 = x0, output_directory = output_dir, max_iter=args.max_iter, save_every=args.save_every, tol=args.tol)

    print("DONE: ")

reason: import numpy,from scipy


text:
#! /usr/bin/env python

import gym
import numpy as np

import matplotlib.pyplot as plt

import rospy

import utils.warning_ignore
from utils.saved_dir import model_dir, log_dir

import sjtu_goto
from stable_baselines.deepq import DQN, MlpPolicy
from stable_baselines import results_plotter
from stable_baselines.results_plotter import load_results, ts2xy
from stable_baselines import PPO2
from stable_baselines.common.evaluation import evaluate_policy
from stable_baselines.common.vec_env import DummyVecEnv, SubprocVecEnv

def main():
    rospy.init_node('train_node', anonymous=True)
    env = gym.make("SJTUGotoEnv-v0")
    env = DummyVecEnv([lambda: env])
    model = PPO2.load(model_dir + "ppo2", env=env)

    obs = env.reset()
    n_steps = 20
    for step in range(n_steps):
        action, _states = model.predict(obs)
        print("Step {}".format(step + 1))
        print("Action: ", action)
        obs, reward, done, info = env.step(action)
        print('obs=', obs, 'reward=', reward, 'done=', done)
        if done:
            print("Goal reached!", "reward=", reward)
            break

if __name__ == '__main__':
    main()


text:
from nas_big_data.combo.multi_inputs_denseskipco import MultiInputsDenseSkipCoFactory

def create_search_space(
    input_shape=[(942,), (3839,), (3839,)], output_shape=(1,), num_layers=5, **kwargs
):
    return MultiInputsDenseSkipCoFactory()(
        input_shape, output_shape, num_layers=num_layers, **kwargs
    )

if __name__ == "__main__":
    # shapes = dict(input_shape=[(8,), (10,)], output_shape=(1,))
    # factory = MultiInputsDenseSkipCoFactory()
    # factory.test(**shapes)
    # factory.plot_model(**shapes)
    # factory.plot_space(**shapes)

    space = create_search_space()
    print(space.size)

# def test_create_search_space():
#     """Generate a random neural network from the search_space definition.
#     """
#     from random import random
#     from tensorflow.keras.utils import plot_model
#     import tensorflow as tf
#     import numpy as np

#     search_space = create_search_space()
#     ops = [
#         9,
#         1,
#         30,
#         1,
#         1,
#         28,
#         1,
#         1,
#         0,
#         24,
#         0,
#         0,
#         1,
#         23,
#         0,
#         1,
#         0,
#         27,
#         1,
#         1,
#         1,
#         18,
#         0,
#         1,
#         1,
#         2,
#         0,
#         1,
#         1,
#         14,
#         1,
#         1,
#         0,
#         20,
#         1,
#         0,
#         1,
#     ]
#     print(ops)
#     print("Search space size: ", search_space.size)

#     print(f"This search_space needs {len(ops)} choices to generate a neural network.")

#     search_space.set_ops(ops)

#     model = search_space.create_model()
#     model.summary()

#     plot_model(model, to_file="sampled_neural_network.png", show_shapes=True)
#     print("The sampled_neural_network.png file has been generated.")

#     # N = 3
#     # shape = (32, 32, 3)

#     # dummy_data = np.random.rand(N, *shape)
#     # y = model.predict(dummy_data)
#     # print(np.shape(y))

# if __name__ == "__main__":
#     test_create_search_space()

max_stars_repo_name: bigwater/NASBigData

max_issues_repo_name: bigwater/NASBigData

max_forks_repo_name: bigwater/NASBigData


text:
import ConfigSpace
import ConfigSpace.hyperparameters
import logging
import numpy as np
import openml
import openmlcontrib
import os
import pandas as pd
import pickle
import sklearn
import typing

def _merge_setup_dict_and_evaluation_dicts(
        setups: typing.Dict[int, openml.setups.OpenMLSetup],
        flow: openml.flows.OpenMLFlow,
        configuration_space: ConfigSpace.ConfigurationSpace,
        evaluations: typing.Dict[str, typing.Dict[int, openml.evaluations.OpenMLEvaluation]],
        per_fold: bool) \
        -> typing.Dict[int, typing.Dict]:
    # returns a dict, mapping from setup id to a dict containing all
    # hyperparameters and evaluation measures
    setup_evaluations = dict()
    for measure in evaluations:
        # evaluations[measure] is a dict mapping from run id to evaluation
        # we can assume that all results are on the same task, so setup is the determining key
        # we will reindex this setup_evaluations[measure] to map from a setup id to evaluation measure
        setup_keys = [eval.setup_id for eval in evaluations[measure].values()]
        task_keys = [eval.task_id for eval in evaluations[measure].values()]
        if len(set(task_keys)) != 1:
            # this should never happen
            raise KeyError('Found multiple task keys in the result set for measure %s' % measure)
        if set(setup_keys) != set(setups.keys()):
            # this should also never happen, and hints at either a bug in setup
            # listing or evaluation listing not complete
            additional = set(setup_keys) - set(setups.keys())
            missing = set(setups.keys()) - set(setup_keys)
            logging.error('Got %d setup records; %d %s records' % (len(setups.keys()), len(setup_keys), measure))
            if additional:
                logging.error('Setup keys additional for %s (%d): %s' % (measure, len(additional), additional))
            if missing:
                logging.error('Setup keys missing for %s (%d): %s' % (measure, len(missing), missing))
            raise KeyError('Setup keys do not align for measure %s' % measure)
        setup_evaluations[measure] = {eval.setup_id: eval for eval in evaluations[measure].values()}
        if len(setup_evaluations[measure]) != len(evaluations[measure]):
            raise KeyError('Lengths of reindexed dict does not comply with old length. ')

    result = dict()
    per_fold_results = None
    for setup in setups.values():
        if setup.flow_id != flow.flow_id:
            # this should never happen
            raise ValueError('Setup and flow do not align.')
        try:
            setup_dict = openmlcontrib.setups.setup_to_parameter_dict(setup=setup,
                                                                      flow=flow,
                                                                      map_library_names=True,
                                                                      configuration_space=configuration_space)
            for measure in evaluations:
                if per_fold:
                    current = setup_evaluations[measure][setup.setup_id].values
                    if per_fold_results is None:
                        per_fold_results = len(current)
                    elif per_fold_results != len(current):
                        raise ValueError('Inconsistent number of per_fold evaluations. Expected %d, got %d' %
                                         (per_fold_results, len(current)))
                    setup_dict[measure] = sum(current) / len(current)
                else:
                    setup_dict[measure] = setup_evaluations[measure][setup.setup_id].value
            result[setup.setup_id] = setup_dict
        except ValueError as e:
            acceptable_errors = ['Trying to set illegal value', 'Active hyperparameter']
            error_acceptable = False
            for acceptable_error in acceptable_errors:
                if e.__str__().startswith(acceptable_error):
                    error_acceptable = True
                    logging.warning('Setup does not comply to configuration space: %s ' % setup.setup_id)
            if not error_acceptable:
                logging.warning('Problem in setup (ValueError will be raised): %s ' % setup.setup_id)
                raise e
    return result

def get_task_flow_results_as_dataframe(task_id: int, flow_id: int,
                                       num_runs: int, raise_few_runs: bool,
                                       configuration_space: ConfigSpace.ConfigurationSpace,
                                       evaluation_measures: typing.List[str],
                                       per_fold: bool,
                                       cache_directory: typing.Union[str, None]) -> pd.DataFrame:
    """
    Obtains a number of runs from a given flow on a given task, and returns a
    (relevant) set of parameters and performance measures. Makes solely use of
    listing functions.

    Parameters
    ----------
    task_id: int
        The task id
    flow_id:
        The flow id
    num_runs: int
        Maximum on the number of runs per task
    raise_few_runs: bool
        Raises an error if not enough runs are found according to the
        `num_runs` argument
    configuration_space: ConfigurationSpace
        Determines valid parameters and ranges. These will be returned as
        column names
    evaluation_measures: List[str]
        A list with the evaluation measure to obtain
    per_fold: bool
        Whether to obtain all results per repeat and per fold (slower, but for
        example run time is not available globally for all runs). Will average
        over these. TODO: add option to get all unaveraged
    cache_directory: optional, str
        Directory where cache files can be stored to or obtained from

    Returns
    -------
    df : pd.DataFrame
        a dataframe with as columns the union of the config_space
        hyperparameters and the evaluation measures, and num_runs rows.
    """
    for measure in evaluation_measures:
        if measure in configuration_space.get_hyperparameters():
            raise ValueError('measure shadows name in hyperparameter list: %s' % measure)
    # both cache paths will be set to a value if cache_directory is not None
    evaluations_cache_path = dict()
    setups_cache_path = None

    # decides the files where the cache will be stored
    if cache_directory is not None:
        cache_flow_task = os.path.join(cache_directory, str(flow_id), str(task_id))
        os.makedirs(cache_flow_task, exist_ok=True)

        for measure in evaluation_measures:
            evaluations_cache_path[measure] = os.path.join(cache_flow_task,
                                                           'evaluations_%s_%d.pkl' % (measure, num_runs))
        setups_cache_path = os.path.join(cache_flow_task, 'setups_%d.pkl' % num_runs)

    # downloads (and caches, if allowed) the evaluations for all measures.
    evaluations = dict()
    setup_ids = set()  # list maintaining all used setup ids
    for measure in evaluation_measures:
        if cache_directory is None or not os.path.isfile(evaluations_cache_path[measure]):
            # downloads (and caches, if allowed) num_runs random evaluations
            evals_current_measure = openml.evaluations.list_evaluations(measure,
                                                                        size=num_runs,
                                                                        task=[task_id],
                                                                        flow=[flow_id],
                                                                        per_fold=per_fold)
            if len(evals_current_measure) < num_runs and raise_few_runs:
                raise ValueError('Not enough evaluations for measure: %s. '
                                 'Required: %d, Got: %d' % (measure, num_runs,
                                                            len(evals_current_measure)))
            if cache_directory is not None and len(evals_current_measure) == num_runs:
                # important to only store cache if enough runs were obtained
                with open(evaluations_cache_path[measure], 'wb') as fp:
                    pickle.dump(evals_current_measure, fp)
            evaluations[measure] = evals_current_measure
        else:
            # obtains the evaluations from cache
            with open(evaluations_cache_path[measure], 'rb') as fp:
                evaluations[measure] = pickle.load(fp)
        if len(evaluations[measure]) == 0:
            raise ValueError('No results on Task %d measure %s according to these criteria' % (task_id, measure))
        for eval in evaluations[measure].values():
            setup_ids.add(eval.setup_id)

    # downloads (and caches, if allowed) the setups that belong to the evaluations
    if cache_directory is None or not os.path.isfile(setups_cache_path):
        setups = openmlcontrib.setups.obtain_setups_by_ids(setup_ids=list(setup_ids))

        if cache_directory is not None and len(setups) == num_runs:
            # important to only store cache if enough runs were obtained
            with open(setups_cache_path, 'wb') as fp:
                pickle.dump(setups, fp)
    else:
        # obtains the setups from cache
        with open(setups_cache_path, 'rb') as fp:
            setups = pickle.load(fp)

    # download flows. Note that only one flow is allowed, per definition
    flows = dict()
    for setup in setups.values():
        if flow_id not in flows:
            flows[setup.flow_id] = openml.flows.get_flow(setup.flow_id)
    if len(flows) != 1:
        # This should never happen.
        raise ValueError('Expected exactly one flow. Got %d' % len(flows))

    # initiates the dataframe object
    relevant_parameters = configuration_space.get_hyperparameter_names()
    all_columns = list(relevant_parameters) + evaluation_measures
    df = pd.DataFrame(columns=all_columns)

    # initiates all records. Note that we need to check them one by one before
    # we can add them to the dataframe
    setups_merged = _merge_setup_dict_and_evaluation_dicts(setups,
                                                           flows[flow_id],
                                                           configuration_space,
                                                           evaluations,
                                                           per_fold)
    # adds the applicable setups to the dataframe
    for setup_id, setup_merged in setups_merged.items():
        # the setups dict still contains the setup objects
        current_setup = setups[setup_id]
        if openmlcontrib.setups.setup_in_config_space(current_setup,
                                                      flows[current_setup.flow_id],
                                                      configuration_space):
            df = df.append(setup_merged, ignore_index=True)
        else:
            logging.warning('Setup does not comply to configuration space: %s ' % setup_id)

    all_numeric_columns = list(evaluation_measures)
    for param in configuration_space.get_hyperparameters():
        if isinstance(param, ConfigSpace.hyperparameters.NumericalHyperparameter):
            all_numeric_columns.append(param.name)

    df[all_numeric_columns] = df[all_numeric_columns].apply(pd.to_numeric)

    if df.shape[0] > num_runs:
        # this should never happen
        raise ValueError('Too many runs. Expected %d got %d' % (num_runs, df.shape[0]))
    exp_params = len(relevant_parameters) + len(evaluation_measures)
    if df.shape[1] != exp_params:
        # this should never happen
        raise ValueError('Wrong number of attributes. Expected %d got %d' % (exp_params, df.shape[1]))
    if df.shape[0] == 0:
        raise ValueError('Did not obtain any results for task %d' % task_id)

    df = df.reindex(sorted(df.columns), axis=1)

    return df

def get_tasks_result_as_dataframe(task_ids: typing.List[int], flow_id: int,
                                  num_runs: int, per_fold: bool, raise_few_runs: bool,
                                  configuration_space: ConfigSpace.ConfigurationSpace,
                                  evaluation_measures: typing.List[str],
                                  normalize: bool,
                                  cache_directory: typing.Optional[str]) -> pd.DataFrame:
    """
    Obtains a number of runs from a given flow on a set of tasks, and returns a
    (relevant) set of parameters and performance measures.

    Parameters
    ----------
    task_ids: List[int]
        The task ids
    flow_id:
        The flow id
    num_runs: int
        Maximum on the number of runs per task
    per_fold: bool
        Whether to obtain all results per repeat and per fold (slower, but for
        example run time is not available globally for all runs). Will average
        over these. TODO: add option to get all unaveraged
    raise_few_runs: bool
        Raises an error if not enough runs are found according to the
        `num_runs` argument
    configuration_space: ConfigurationSpace
        Determines valid parameters and ranges. These will be returned as
        column names
    evaluation_measures: List[str]
        A list with the evaluation measure to obtain
    normalize: bool
        Whether to normalize the measures per task to interval [0,1]
    cache_directory: optional, str
        Directory where cache files can be stored to or obtained from. Only
        relevant when per_fold is True

    Returns
    -------
    df : pd.DataFrame
        a dataframe with as columns the union of the config_space
        hyperparameters and the evaluation measures, and num_runs rows.
    """
    setup_data_all = None
    scaler = sklearn.preprocessing.MinMaxScaler()
    for idx, task_id in enumerate(task_ids):
        logging.info('Currently processing task %d (%d/%d)' % (task_id, idx+1, len(task_ids)))
        try:

            setup_data = get_task_flow_results_as_dataframe(task_id=task_id,
                                                            flow_id=flow_id,
                                                            num_runs=num_runs,
                                                            raise_few_runs=raise_few_runs,
                                                            configuration_space=configuration_space,
                                                            evaluation_measures=evaluation_measures,
                                                            cache_directory=cache_directory,
                                                            per_fold=per_fold)
        except openml.exceptions.OpenMLServerException as e:
            if raise_few_runs:
                raise e
            logging.warning('OpenMLServerException in Task %d: %s' % (task_id, str(e)))
            continue
        except ValueError as e:
            if raise_few_runs:
                raise e
            logging.warning('ValueError in Task %d: %s' % (task_id, str(e)))
            continue
        setup_data['task_id'] = task_id
        logging.info('Obtained result frame with dimensions %s' % str(setup_data.shape))
        if normalize:
            for measure in evaluation_measures:
                setup_data[[measure]] = scaler.fit_transform(setup_data[[measure]])
        if setup_data_all is None:
            setup_data_all = setup_data
        else:
            if list(setup_data.columns.values) != list(setup_data_all.columns.values):
                raise ValueError('Columns per task result do not match')
            setup_data_all = pd.concat((setup_data_all, setup_data))
    if setup_data_all is None:
        raise ValueError('Results for None of the tasks obtained successfully')
    return setup_data_all

def get_tasks_qualities_as_dataframe(task_ids: typing.List[int],
                                     normalize: bool,
                                     impute_nan_value: float,
                                     drop_missing: bool,
                                     raise_missing_task: bool) -> pd.DataFrame:
    """
    Obtains all meta-features from a given set of tasks. Meta-features that are
    calculated but not applicable for a given task (e.g., MutualInformation for
    numeric-only datasets) can be imputed, meta-features that are not calculated
    on all datasets can be dropped.

    Parameters
    ----------
    task_ids: List[int]
        The task ids

    normalize: bool
        Whether to normalize all entrees per column to the interval [0, 1]

    impute_nan_value: float
        The value to impute non-applicable meta-features with

    drop_missing: bool
        Whether to drop all meta-features that are not calculated on all tasks

    raise_missing_task: bool
        If set to true, an error is raised when one of the tasks does not have meta-features

    Returns
    -------
    result: pd.DataFrame
        Dataframe with for each task a row and per meta-feature a column
    """
    def scale(val, min_val, max_val):
        return (val - min_val) / (max_val - min_val)

    task_qualities = dict()
    task_nanqualities = dict()
    tasks = openml.tasks.list_tasks(task_id=task_ids, status='all')
    for idx, task_id in enumerate(task_ids):
        logging.info('Obtaining qualities for task %d (%d/%d)' % (task_id, idx + 1, len(task_ids)))
        try:
            dataset = openml.datasets.get_dataset(tasks[task_id]['did'])
            qualities = dataset.qualities
            # nanqualities are qualities that are calculated, but not-applicable
            task_nanqualities[task_id] = {k for k, v in qualities.items() if np.isnan(v)}
            task_qualities[task_id] = dict(qualities.items())
        except openml.exceptions.OpenMLServerException as e:
            if raise_missing_task or e.code != 274:
                raise e
            else:
                logging.warning(e.message)
    # index of qualities: the task id
    qualities_frame = pd.DataFrame.from_dict(task_qualities, orient='index', dtype=np.float)
    if normalize:
        for quality in qualities_frame.columns.values:
            min_val = min(qualities_frame[quality])
            max_val = max(qualities_frame[quality])
            if min_val == max_val:
                logging.warning('Quality can not be normalized, as it is constant: %s' % quality)
                continue
            qualities_frame[quality] = qualities_frame[quality].apply(lambda x: scale(x, min_val, max_val))
    # now qualities are all in the range [0, 1], set, reset the values of qualities
    for task_id in qualities_frame.index.values:
        for quality in task_nanqualities[task_id]:
            qualities_frame.at[task_id, quality] = impute_nan_value

    if drop_missing:
        qualities_frame = pd.DataFrame.dropna(qualities_frame, axis=1, how='any')
    return qualities_frame

def dataframe_to_arff(dataframe, relation, description):
    attributes = []
    for idx, column_name in enumerate(dataframe.columns.values):
        if np.issubdtype(dataframe[column_name].dtype, np.number):
            attributes.append((column_name, 'NUMERIC'))
        else:
            values = dataframe[column_name].unique()
            attributes.append((column_name, [str(value) for value in values]))

    arff_dict = dict()
    arff_dict['data'] = dataframe.values
    arff_dict['attributes'] = attributes
    arff_dict['description'] = description
    arff_dict['relation'] = relation

    return arff_dict

def arff_to_dataframe(liac_arff_dict: typing.Dict,
                      config_space: typing.Optional[ConfigSpace.ConfigurationSpace]=None):
    """
    Transforms a file, as loaded by liac arff, to a pandas data frame.
    In case a config space object is provided, it is assumed that all
    hyperparameters in the config space object are also present in the arff
    dict. Those columns will be casted as their corresponding datatype

    TODO: doc and unit tests
    """
    def post_process(vals: np.array, col_name: str) -> np.array:
        if config_space is not None:
            if col_name in config_space.get_hyperparameter_names():
                if openmlcontrib.legacy.is_boolean_hyperparameter(
                        config_space.get_hyperparameter(col_name)):
                    vals = vals == 'True'
        return vals

    pd_extension_int = 'Int64'

    numeric_keywords = {'numeric', 'real'}

    expected_keys = {'data', 'attributes', 'description', 'relation'}
    if liac_arff_dict.keys() != expected_keys:
        raise ValueError('liacarff object does not contain correct keys.')
    data_ = np.array(liac_arff_dict['data'])
    col_names = [col_name for col_name, col_type in liac_arff_dict['attributes']]

    column_dtypes = {
        # str(col_type).lower() important because there are also lists, and uppercase statements
        col_name: np.float64 if str(col_type).lower() in numeric_keywords else np.dtype(object)
        for idx, (col_name, col_type) in enumerate(liac_arff_dict['attributes'])
    }

    if config_space is not None:
        for hyperparameter in config_space.get_hyperparameters():
            if hyperparameter.name not in column_dtypes.keys():
                raise ValueError('ConfigSpace does not align with meta-data. '
                                 'Missing in arff: %s' % hyperparameter.name)
            if openmlcontrib.legacy.is_boolean_hyperparameter(hyperparameter):
                column_dtypes[hyperparameter.name] = bool
            elif openmlcontrib.legacy.is_integer_hyperparameter(hyperparameter):
                column_dtypes[hyperparameter.name] = pd_extension_int

    # can break, if integer encoded as string float
    arff_dict = {
        col_name: pd.Series(np.array(data_[:, idx], dtype=np.float64)
                            if column_dtypes[col_name] == pd_extension_int
                            else post_process(data_[:, idx], col_name), dtype=column_dtypes[col_name])
        for idx, (col_name, _) in enumerate(liac_arff_dict['attributes'])
    }

    result = pd.DataFrame(arff_dict)
    # now reset original column order
    result = result[col_names]
    return result

def integer_encode_dataframe(df: pd.DataFrame,
                             config_space: ConfigSpace.ConfigurationSpace) -> pd.DataFrame:
    """
    Takes a dataframe and encodes all columns whose name aligns with a
    Constant,  UnparameterzedHyperparameter or CategoricalHyperparameter as
    integer.

    Parameters
    ----------
    df: pd.DataFrame
        the dataframe to encode

    config_space: ConfigSpace.ConfigurationSpace
        the configuration space. Each hyperparameter that has a name
        corresponding with a dataframe column will be encoded appropriately.

    Returns
    -------
    pd.DataFrame
        a dataframe with (for all hyperparameters) integer encoded variables
    """
    for column_name in df.columns.values:
        if column_name in config_space.get_hyperparameter_names():
            hyperparameter = config_space.get_hyperparameter(column_name)
            if isinstance(hyperparameter, ConfigSpace.hyperparameters.NumericalHyperparameter):
                # numeric hyperparameter, don't do anything
                pass
            elif isinstance(hyperparameter, (ConfigSpace.hyperparameters.Constant,
                                             ConfigSpace.hyperparameters.UnParametrizedHyperparameter)):
                # encode as constant value. can be retrieved from config space later
                df[column_name] = 0
                df[column_name] = pd.to_numeric(df[column_name])
            elif isinstance(hyperparameter, ConfigSpace.hyperparameters.CategoricalHyperparameter):
                df[column_name] = df[column_name].apply(lambda x: hyperparameter.choices.index(x))
                df[column_name] = pd.to_numeric(df[column_name])
            else:
                raise NotImplementedError('Function not implemented for '
                                          'Hyperparameter: %s' % type(hyperparameter))
    return df

def decode_integer_dataframe(df: pd.DataFrame,
                             config_space: ConfigSpace.ConfigurationSpace) -> pd.DataFrame:
    """
    Takes a dataframe and decodes all columns whose name aligns with a
    Constant,  UnparameterzedHyperparameter or CategoricalHyperparameter as
    back to the original value.

    Parameters
    ----------
    df: pd.DataFrame
        the dataframe to encode

    config_space: ConfigSpace.ConfigurationSpace
        the configuration space. Each hyperparameter that has a name
        corresponding with a dataframe column will be decoded to the original
        value.

    Returns
    -------
    pd.DataFrame
        a dataframe with (for all hyperparameters) decoded variables
    """
    for column_name in df.columns.values:
        if column_name in config_space.get_hyperparameter_names():
            hyperparameter = config_space.get_hyperparameter(column_name)
            if isinstance(hyperparameter, ConfigSpace.hyperparameters.NumericalHyperparameter):
                # numeric hyperparameter, don't do anything
                pass
            elif isinstance(hyperparameter, (ConfigSpace.hyperparameters.Constant,
                                             ConfigSpace.hyperparameters.UnParametrizedHyperparameter)):
                # encode as constant value. can be retrieved from config space later
                if df[column_name].dtype != np.int64:
                    raise ValueError('Expected integer datatype for column: %s. Got: %s' % (column_name, df[column_name].dtype))
                df[column_name] = hyperparameter.value
            elif isinstance(hyperparameter, ConfigSpace.hyperparameters.CategoricalHyperparameter):
                if df[column_name].dtype != np.int64:
                    raise ValueError('Expected integer datatype for column: %s. Got: %s' % (column_name, df[column_name].dtype))
                df[column_name] = df[column_name].apply(lambda x: hyperparameter.choices[x])
            else:
                raise NotImplementedError('Function not implemented for '
                                          'Hyperparameter: %s' % type(hyperparameter))
    return df

max_stars_repo_name: openml/openml-python-contrib

max_issues_repo_name: openml/openml-python-contrib

max_forks_repo_name: openml/openml-python-contrib


text:
import os
import numpy as np

def _download_and_extract(url, path, filename):
    import shutil, zipfile
    import requests

    fn = os.path.join(path, filename)

    while True:
        try:
            with zipfile.ZipFile(fn) as zf:
                zf.extractall(path)
            print('Unzip finished.')
            break
        except Exception:
            os.makedirs(path, exist_ok=True)
            f_remote = requests.get(url, stream=True)
            sz = f_remote.headers.get('content-length')
            assert f_remote.status_code == 200, 'fail to open {}'.format(url)
            with open(fn, 'wb') as writer:
                for chunk in f_remote.iter_content(chunk_size=1024*1024):
                    writer.write(chunk)
            print('Download finished. Unzipping the file...')

class KGDataset1:
    '''Load a knowledge graph with format 1

    In this format, the folder with a knowledge graph has five files:
    * entities.dict stores the mapping between entity Id and entity name.
    * relations.dict stores the mapping between relation Id and relation name.
    * train.txt stores the triples in the training set.
    * valid.txt stores the triples in the validation set.
    * test.txt stores the triples in the test set.

    The mapping between entity (relation) Id and entity (relation) name is stored as 'id\tname'.

    The triples are stored as 'head_name\trelation_name\ttail_name'.
    '''
    def __init__(self, path, name, read_triple=True, only_train=False):
        url = 'https://s3.us-east-2.amazonaws.com/dgl.ai/dataset/{}.zip'.format(name)

        if not os.path.exists(os.path.join(path, name)):
            print('File not found. Downloading from', url)
            _download_and_extract(url, path, name + '.zip')
        path = os.path.join(path, name)

        with open(os.path.join(path, 'entities.dict')) as f:
            entity2id = {}
            for line in f:
                eid, entity = line.strip().split('\t')
                entity2id[entity] = int(eid)

        self.entity2id = entity2id

        with open(os.path.join(path, 'relations.dict')) as f:
            relation2id = {}
            for line in f:
                rid, relation = line.strip().split('\t')
                relation2id[relation] = int(rid)

        self.relation2id = relation2id

        # TODO: to deal with contries dataset.

        self.n_entities = len(self.entity2id)
        self.n_relations = len(self.relation2id)

        if read_triple == True:
            self.train = self.read_triple(path, 'train')
            if only_train == False:
                self.valid = self.read_triple(path, 'valid')
                self.test = self.read_triple(path, 'test')

    def read_triple(self, path, mode):
        # mode: train/valid/test
        heads = []
        tails = []
        rels = []
        with open(os.path.join(path, '{}.txt'.format(mode))) as f:
            for line in f:
                h, r, t = line.strip().split('\t')
                heads.append(self.entity2id[h])
                rels.append(self.relation2id[r])
                tails.append(self.entity2id[t])
        heads = np.array(heads, dtype=np.int64)
        tails = np.array(tails, dtype=np.int64)
        rels = np.array(rels, dtype=np.int64)

        return (heads, rels, tails)

class KGDataset2:
    '''Load a knowledge graph with format 2

    In this format, the folder with a knowledge graph has five files:
    * entity2id.txt stores the mapping between entity name and entity Id.
    * relation2id.txt stores the mapping between relation name relation Id.
    * train.txt stores the triples in the training set.
    * valid.txt stores the triples in the validation set.
    * test.txt stores the triples in the test set.

    The mapping between entity (relation) name and entity (relation) Id is stored as 'name\tid'.

    The triples are stored as 'head_nid\trelation_id\ttail_nid'.
    '''
    def __init__(self, path, name, read_triple=True, only_train=False):
        url = 'https://s3.us-east-2.amazonaws.com/dgl.ai/dataset/{}.zip'.format(name)

        if not os.path.exists(os.path.join(path, name)):
            print('File not found. Downloading from', url)
            _download_and_extract(url, path, '{}.zip'.format(name))
        self.path = os.path.join(path, name)

        f_rel2id = os.path.join(self.path, 'relation2id.txt')
        with open(f_rel2id) as f_rel:
            self.n_relations = int(f_rel.readline()[:-1])

        if only_train == True:
            f_ent2id = os.path.join(self.path, 'local_to_global.txt')
            with open(f_ent2id) as f_ent:
                self.n_entities = len(f_ent.readlines())
        else:
            f_ent2id = os.path.join(self.path, 'entity2id.txt')
            with open(f_ent2id) as f_ent:
                self.n_entities = int(f_ent.readline()[:-1])

        if read_triple == True:
            self.train = self.read_triple(self.path, 'train')
            if only_train == False:
                self.valid = self.read_triple(self.path, 'valid')
                self.test = self.read_triple(self.path, 'test')

    def read_triple(self, path, mode, skip_first_line=False):
        heads = []
        tails = []
        rels = []
        print('Reading {} triples....'.format(mode))
        with open(os.path.join(path, '{}.txt'.format(mode))) as f:
            if skip_first_line:
                _ = f.readline()
            for line in f:
                h, t, r = line.strip().split('\t')
                heads.append(int(h))
                tails.append(int(t))
                rels.append(int(r))
        heads = np.array(heads, dtype=np.int64)
        tails = np.array(tails, dtype=np.int64)
        rels = np.array(rels, dtype=np.int64)
        print('Finished. Read {} {} triples.'.format(len(heads), mode))
        return (heads, rels, tails)

def get_dataset(data_path, data_name, format_str):
    if data_name == 'Freebase':
        dataset = KGDataset2(data_path, data_name)
    elif format_str == '1':
        dataset = KGDataset1(data_path, data_name)
    else:
        dataset = KGDataset2(data_path, data_name)

    return dataset

def get_partition_dataset(data_path, data_name, format_str, part_id):
    part_name = os.path.join(data_name, 'part_'+str(part_id))

    if data_name == 'Freebase':
        dataset = KGDataset2(data_path, part_name, read_triple=True, only_train=True)
    elif format_str == '1':
        dataset = KGDataset1(data_path, part_name, read_triple=True, only_train=True)
    else:
        dataset = KGDataset2(data_path, part_name, read_triple=True, only_train=True)

    path = os.path.join(data_path, part_name)

    partition_book = []
    with open(os.path.join(path, 'partition_book.txt')) as f:
        for line in f:
            partition_book.append(int(line))

    local_to_global = []
    with open(os.path.join(path, 'local_to_global.txt')) as f:
        for line in f:
            local_to_global.append(int(line))

    return dataset, partition_book, local_to_global

def get_server_partition_dataset(data_path, data_name, format_str, part_id):
    part_name = os.path.join(data_name, 'part_'+str(part_id))

    if data_name == 'Freebase':
        dataset = KGDataset2(data_path, part_name, read_triple=False, only_train=True)
    elif format_str == '1':
        dataset = KGDataset1(data_path, part_name, read_triple=False, only_train=True)
    else:
        dataset = KGDataset2(data_path, part_name, read_triple=False, only_train=True)

    path = os.path.join(data_path, part_name)

    n_entities = len(open(os.path.join(path, 'partition_book.txt')).readlines())

    local_to_global = []
    with open(os.path.join(path, 'local_to_global.txt')) as f:
        for line in f:
            local_to_global.append(int(line))

    global_to_local = [0] * n_entities
    for i in range(len(local_to_global)):
        global_id = local_to_global[i]
        global_to_local[global_id] = i

    local_to_global = None

    return global_to_local, dataset

max_stars_repo_name: jspisak/dgl

max_issues_repo_name: echomint/dgl

max_forks_repo_name: echomint/dgl


text:
import sys
import numpy as np
from os.path import join as opj
from PIL import Image, ImageDraw, ImageFont

sys.path.insert(0, '/home/nik/Git/blitznet/')

from paths import EVAL_DIR

VOC_CATS = ['__background__', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle',
            'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse',
            'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train',
            'tvmonitor']

colors = np.load(opj(EVAL_DIR, 'Extra/colors.npy')).tolist()
font = ImageFont.truetype(opj(EVAL_DIR, "Extra/FreeSansBold.ttf"), 20)

size = 500
step = (size - 100) // 10
canvas = Image.new('RGB', (size, size), (255, 255, 255))
dr = ImageDraw.Draw(canvas)

for i in range(1, 21):
    corner = (100 + size / 2 * (i > 10), 50 * ((i - 1) % 10) + 50)
    category = VOC_CATS[i]
    print(category, corner)
    color = colors[i-1]
    dr.text(corner, category, fill=color, font=font)

canvas.show()

max_stars_repo_name: exmee/HSSD

max_issues_repo_name: exmee/HSSD

max_forks_repo_name: exmee/HSSD


text:
import logging
import numpy as np

import torch
from torch.nn import functional as F

from .featurizer import Featurizer
from holoclean.dataset import AuxTables
from holoclean.domain.estimators import TupleEmbedding

class EmbeddingFeaturizer(Featurizer):
    """
    Featurizer that wraps the learned vectors from a TupleEmbedding model
    cached in Dataset.

    Takes in two optional parameters (if estimator_type != TupleEmbedding i.e.
    if estimator is not the TupleEmbedding estimator):
        epochs (int): # of epochs to run
        batch_size (int): batch size
    """
    DEFAULT_EPOCHS = 10
    DEFAULT_BATCH_SIZE = 32
    DEFAULT_DROPOUT = 0.
    DEFAULT_LR = 0.05
    DEFAULT_WEIGHT_LAMBDA = 0.

    def specific_setup(self):
        self.name = 'EmbeddingFeaturizer'

        if self.env['estimator_type'] != 'TupleEmbedding':
            logging.debug('%s: training TupleEmbedding model since it was not used as an estimator',
                    self.name)

            epochs = self.addn_kwargs.get('epochs', self.DEFAULT_EPOCHS)
            batch_size = self.addn_kwargs.get('batch_size', self.DEFAULT_BATCH_SIZE)
            dropout_pct = self.addn_kwargs.get('dropout_pct', self.DEFAULT_DROPOUT)
            weight_lambda = self.addn_kwargs.get('weight_lambda', self.DEFAULT_WEIGHT_LAMBDA)
            learning_rate = self.addn_kwargs.get('learning_rate', self.DEFAULT_LR)
            numerical_attr_groups = self.addn_kwargs.get('numerical_attr_groups', None)
            validate_fpath = self.addn_kwargs.get('validate_fpath', None)
            validate_tid_col = self.addn_kwargs.get('validate_tid_col', 'tid')
            validate_attr_col = self.addn_kwargs.get('validate_attr_col', 'attribute')
            validate_val_col = self.addn_kwargs.get('validate_val_col', 'correct_val')
            validate_epoch = self.addn_kwargs.get('validate_epoch', 1)
            reuse_embedding = self.addn_kwargs.get('reuse_embedding', False)

            logging.debug('%s: training with %d epochs and %d batch size',
                          self.name,
                          epochs,
                          batch_size)

            domain_df = self.ds.aux_table[AuxTables.cell_domain].df.sort_values('_vid_')
            self.embedding_model = TupleEmbedding(self.env, self.ds, domain_df,
                                                  numerical_attr_groups=numerical_attr_groups,
                                                  dropout_pct=dropout_pct,
                                                  learning_rate=learning_rate,
                                                  validate_fpath=validate_fpath,
                                                  validate_tid_col=validate_tid_col,
                                                  validate_attr_col=validate_attr_col,
                                                  validate_val_col=validate_val_col,
                                                  validate_epoch=validate_epoch,
                                                  reuse_embedding=reuse_embedding
                                                  )

            dump_prefix = self.addn_kwargs.get('dump_prefix', None)
            if dump_prefix is None \
                    or not self.embedding_model.load_model(dump_prefix):
                self.embedding_model.train(batch_size, weight_entropy_lambda=weight_lambda)

                if dump_prefix is not None:
                    self.embedding_model.dump_model(dump_prefix)

            logging.debug('%s: done training TupleEmbedding model.', self.name)
        else:
            logging.debug('%s: retrieving embedding vectors learned from TupleEmbedding estimator',
                    self.name)
            self.embedding_model = self.ds.get_embedding_model()

    def create_tensor(self):
        """
        For a batch of vids, returns a batch of softmax probabilities for each
        domain value learned by an embedding model.

        That is returns a (batch, max domain, 1) tensor.
        """
        domain_df = self.ds.aux_table[AuxTables.cell_domain].df.sort_values('_vid_').reset_index(drop=True)

        vids = domain_df['_vid_']

        # (# of vids, max_cat_domain), (# of vids, 1), (# of vids, 1)
        cat_probas, num_predvals, is_categorical = self.embedding_model.get_features(vids)

        # (# of vids, max domain)
        pad_len = self.embedding_model.max_domain - self.embedding_model.max_cat_domain
        if pad_len:
            # Pad last dimension on the right side with pad_len
            cat_probas = F.pad(cat_probas, pad=(0,pad_len), mode='constant', value=0.)

        # Create tensor for z-scored domain values
        num_attrs_idx = {attr: idx for idx, attr in enumerate(self.embedding_model._train_num_attrs)}
        domain_numvals = torch.zeros(len(vids), self.embedding_model.max_domain, len(num_attrs_idx))
        # only num_attrs_idx[attr] makes sense, others should not count into num_rmse
        pred_numvals = torch.zeros(len(vids), self.embedding_model.max_domain, len(num_attrs_idx))
        # Mask to mask out RMSE computed on padding outside of cell's domain.
        domain_mask = torch.zeros(len(vids), self.embedding_model.max_domain)
        for idx, (_, attr, domain, domain_sz) in enumerate(domain_df[['attribute', 'domain', 'domain_size']].to_records()):
            if domain_sz == 0:
                continue
            if is_categorical[idx,0] == 1.:
                continue
            dom_arr = np.array(domain.split('|||'), dtype=np.float32)
            mean = self.embedding_model._dataset._num_attrs_mean[attr]
            std = self.embedding_model._dataset._num_attrs_std[attr]
            dom_arr = (dom_arr - mean) / std
            domain_numvals[idx,:domain_sz, num_attrs_idx[attr]] = torch.FloatTensor(dom_arr)
            domain_mask[idx,:domain_sz] = 1.

            num_pred = num_predvals[idx]
            pred_numvals[idx, :domain_sz, num_attrs_idx[attr]] = torch.FloatTensor(num_pred)

        # (# of vids, max domain, # of num attrs)
        # This RMSE is between z-scored values. This is equivalent to dividing
        # the RMSE by std^2.
        # num_rmse = torch.abs(num_predvals.unsqueeze(-1).expand(-1, self.embedding_model.max_domain, len(num_attrs_idx)) - domain_numvals)
        num_rmse = torch.abs(pred_numvals - domain_numvals)
        num_rmse.mul_(domain_mask.unsqueeze(-1).expand(-1, -1, len(num_attrs_idx)))

        # (# of vids, max domain, 1 + # num attrs)
        return torch.cat([cat_probas.unsqueeze(-1), num_rmse], dim=-1)

    def feature_names(self):
        return ["Embedding Cat Proba"]  + ["Embedding Num RMSE (%s)" % attr for attr in self.embedding_model._train_num_attrs]

max_stars_repo_name: cgebest/holoclean

max_issues_repo_name: cgebest/holoclean

max_forks_repo_name: cgebest/holoclean


text:
import kerastuner
import numpy as np

import autokeras as ak
from autokeras import graph as graph_module
from autokeras import nodes as input_module
from autokeras.hypermodels import heads as head_module

def test_two_classes():
    y = np.array(['a', 'a', 'a', 'b'])
    head = head_module.ClassificationHead(name='a')
    adapter = head.get_adapter()
    adapter.fit_transform(y)
    head.config_from_adapter(adapter)
    head.output_shape = (1,)
    head.build(kerastuner.HyperParameters(), input_module.Input(shape=(32,)).build())
    assert head.loss == 'binary_crossentropy'

def test_three_classes():
    y = np.array(['a', 'a', 'c', 'b'])
    head = head_module.ClassificationHead(name='a')
    adapter = head.get_adapter()
    adapter.fit_transform(y)
    head.config_from_adapter(adapter)
    assert head.loss == 'categorical_crossentropy'

def test_segmentation():
    y = np.array(['a', 'a', 'c', 'b'])
    head = head_module.SegmentationHead(name='a')
    adapter = head.get_adapter()
    adapter.fit_transform(y)
    head.config_from_adapter(adapter)
    input_shape = (64, 64, 21)
    hp = kerastuner.HyperParameters()
    head = graph_module.deserialize(graph_module.serialize(head))
    head.build(hp, ak.Input(shape=input_shape).build())

max_stars_repo_name: jkhenning/autokeras

max_issues_repo_name: achaar/autokeras

max_forks_repo_name: achaar/autokeras


text:
# https://quera.ir/problemset/contest/2887/
import numpy as np

n = int(input())
A_str = input().split(' ')
A = list(map(int, A_str))

addiction_level = np.gcd.reduce(A)
result = 0
for val in A:
    temp = val / addiction_level
    result += temp

print(int(result))

max_stars_repo_name: mhdehghan/quera-answers

max_issues_repo_name: mhdehghan/quera-answers

max_forks_repo_name: mhdehghan/quera-answers


text:
import os
import shutil
import numpy as np
import sys

from .. import units

from .qmbase import QMBase
from ..qmtmpl import QMTmpl

class ORCA(QMBase):

    QMTOOL = 'ORCA'

    def get_mm_system(self, embed):
        """Load MM information."""

        super(ORCA, self).get_mm_system(embed)

        self._n_mm_atoms = self.mm_atoms_near.n_atoms
        self._mm_position = self.mm_atoms_near.position
        self._mm_charge = self.mm_atoms_near.charge_eed

        if self.mm_atoms_far.charge_eeq is not None:
            self._qm_esp_near = embed.qm_esp_near
            self._qm_efield_near = embed.qm_efield_near
            raise NotImplementedError()

    def get_qm_params(self, method=None, basis=None, **kwargs):
        """Get the parameters for QM calculation."""

        super(ORCA, self).get_qm_params(**kwargs)

        if method is not None:
            self.method = method
        else:
            raise ValueError("Please set method for ORCA.")

        if basis is not None:
            self.basis = basis
        else:
            raise ValueError("Please set basis for ORCA.")

    def gen_input(self, path=None):
        """Generate input file for QM software."""

        qmtmpl = QMTmpl(self.QMTOOL)

        if self.calc_forces:
            calc_forces = 'EnGrad '
        else:
            calc_forces = ''

        if self.read_guess:
            read_guess = ''
        else:
            read_guess = 'NoAutoStart '

        if self.addparam is not None:
            if isinstance(self.addparam, list):
                addparam = "".join(["%s " % i for i in self.addparam])
            else:
                addparam = self.addparam + " "
        else:
            addparam = ''

        nproc = self.get_nproc()

        if path is None:
            path = self.basedir

        with open(os.path.join(path, "orca.inp"), 'w') as f:
            f.write(qmtmpl.gen_qmtmpl().substitute(
                method=self.method, basis=self.basis,
                calc_forces=calc_forces, read_guess=read_guess,
                addparam=addparam, nproc=nproc))
            f.write("%pointcharges \"orca.pntchrg\"\n")
            f.write("%coords\n")
            f.write("  CTyp xyz\n")
            f.write("  Charge %d\n" % self.charge)
            f.write("  Mult %d\n" % self.mult)
            f.write("  Units Angs\n")
            f.write("  coords\n")

            for i in range(self._n_qm_atoms):
                f.write(" ".join(["%6s" % self._qm_element[i],
                                  "%22.14e" % self._qm_position[i, 0],
                                  "%22.14e" % self._qm_position[i, 1],
                                  "%22.14e" % self._qm_position[i, 2], "\n"]))
            f.write("  end\n")
            f.write("end\n")

        with open(os.path.join(path, "orca.pntchrg"), 'w') as f:
            f.write("%d\n" % self._n_mm_atoms)
            for i in range(self._n_mm_atoms):
                f.write("".join(["%22.14e " % self._mm_charge[i],
                                 "%22.14e" % self._mm_position[i, 0],
                                 "%22.14e" % self._mm_position[i, 1],
                                 "%22.14e" % self._mm_position[i, 2], "\n"]))

        with open(os.path.join(path, "orca_pol_corr.inp"), 'w') as f:
            f.write(qmtmpl.gen_qmtmpl().substitute(
                method=self.method, basis=self.basis,
                calc_forces=calc_forces, read_guess="",
                addparam=addparam, nproc=nproc))
            f.write("%coords\n")
            f.write("  CTyp xyz\n")
            f.write("  Charge %d\n" % self.charge)
            f.write("  Mult %d\n" % self.mult)
            f.write("  Units Angs\n")
            f.write("  coords\n")

            for i in range(self._n_qm_atoms):
                f.write(" ".join(["%6s" % self._qm_element[i],
                                  "%22.14e" % self._qm_position[i, 0],
                                  "%22.14e" % self._qm_position[i, 1],
                                  "%22.14e" % self._qm_position[i, 2], "\n"]))
            f.write("  end\n")
            f.write("end\n")

        with open(os.path.join(path, "orca.pntvpot.xyz"), 'w') as f:
            f.write("%d\n" % self._n_mm_atoms)
            for i in range(self._n_mm_atoms):
                f.write("".join(["%22.14e" % (self._mm_position[i, 0] / units.L_AU),
                                 "%22.14e" % (self._mm_position[i, 1] / units.L_AU),
                                 "%22.14e" % (self._mm_position[i, 2] / units.L_AU), "\n"]))

    def gen_cmdline(self):
        """Generate commandline for QM calculation."""

        cmdline = "cd " + self.basedir + "; "
        # Check if orca is in the PATH
        executables_dir = os.environ['PATH']
        for executable in executables_dir.split(":"):
            if 'ORCA' in executable:
                orca_cmd= executable + "/orca"
                cmdline += orca_cmd + " orca.inp > orca.out; cp orca.gbw orca_pol_corr.gbw; "
                cmdline += orca_cmd + "_2mkl" + " orca -molden > /dev/null; "
                cmdline += orca_cmd + "_vpot" + " orca.gbw orca.scfp orca.pntvpot.xyz orca.pntvpot.out >> orca_vpot.out; "
                cmdline += orca_cmd + " orca_pol_corr.inp > orca_pol_corr.out "
                return cmdline
            else:
                orca_cmd = ""
        if orca_cmd == "":
            print('No orca executable in $PATH')
            sys.exit(1)

    def rm_guess(self):
        """Remove save from previous QM calculation."""

        qmsave = os.path.join(self.basedir, "orca.gbw")
        if os.path.isfile(qmsave):
            os.remove(qmsave)

    def parse_output(self):
        """Parse the output of QM calculation."""

        output = self.load_output(os.path.join(self.basedir, "orca.out"))

        self.get_qm_energy(output)
        self.get_qm_charge(output)

        output_pol_corr = self.load_output(os.path.join(self.basedir, "orca_pol_corr.out"))
        self.get_pol_energy(output_pol_corr)

        self.get_mm_esp_eed()

        self.qm_atoms.qm_energy = self.qm_energy * units.E_AU
        self.qm_atoms.qm_pol_energy = self.qm_pol_energy * units.E_AU
        self.qm_atoms.qm_charge = self.qm_charge
        if self.calc_forces is True:
            self.get_qm_force()
            self.get_mm_force()
            self.qm_atoms.force = self.qm_force * units.F_AU
            self.mm_atoms_near.force = self.mm_force * units.F_AU

        self.mm_atoms_near.esp_eed = self.mm_esp_eed * units.E_AU

    def get_qm_energy(self, output=None):
        """Get QM energy from output of QM calculation."""

        if output is None:
            output = self.load_output(os.path.join(self.basedir, "orca.out"))

        for line in output:
            line = line.strip().expandtabs()
            if "FINAL SINGLE POINT ENERGY" in line:
                self.qm_energy = float(line.split()[-1])
                break

        return self.qm_energy

    def get_pol_energy(self, output_pol_corr=None):
        """Get Polarization energy from output of QM calculation."""

        if output_pol_corr is None:
            output_pol_corr = self.load_output(os.path.join(self.basedir, "orca_pol_corr.out"))

        read = False
        for line in output_pol_corr:
            line = line.strip().expandtabs()
            if read:
                words = line.split()
                if words[0] == "0":
                    esolv = float(line.split()[1])
                    read = False

            if line.startswith("***  Starting"):
                read = True

            if "FINAL SINGLE POINT ENERGY" in line:
                egas = float(line.split()[-1])
                break

        self.qm_pol_energy = esolv - egas
        return self.qm_pol_energy

    def get_qm_charge(self, output=None):
        """Get Mulliken charges from output of QM calculation."""

        if output is None:
            output = self.load_output(os.path.join(self.basedir, "orca.out"))

        for i in range(len(output)):
            if "MULLIKEN ATOMIC CHARGES" in output[i]:
                charges = []
                for line in output[(i + 2):(i + 2 + self._n_qm_atoms)]:
                    charges.append(float(line.split()[-1]))
                break

        self.qm_charge = np.array(charges)

        return self.qm_charge

    def get_qm_force(self, output=None):
        """Get QM forces from output of QM calculation."""

        if output is None:
            output = self.load_output(os.path.join(self.basedir, "orca.engrad"))

        start = 11
        stop = start + self._n_qm_atoms * 3
        self.qm_force = -1 * np.loadtxt(output[start:stop]).reshape((self._n_qm_atoms, 3))

        return self.qm_force

    def get_mm_force(self, output=None):
        """Get external point charge forces from output of QM calculation."""

        if output is None:
            output = self.load_output(os.path.join(self.basedir, "orca.pcgrad"))

        self.mm_force = -1 * np.loadtxt(output[1:(self._n_mm_atoms + 1)])

        return self.mm_force

    def get_mm_esp_eed(self, output=None):
        """Get ESP at MM atoms in the near field from QM density."""

        if output is None:
            output = self.load_output(os.path.join(self.basedir, "orca.pntvpot.out"))

        self.mm_esp_eed = np.loadtxt(output[1:(self._n_mm_atoms + 1)], usecols=3)

        return self.mm_esp_eed

max_stars_repo_name: QCMM/dynqmprop

max_issues_repo_name: QCMM/dynqmprop

max_forks_repo_name: QCMM/dynqmprop


text:
import numpy as np
from model import generate_recommendations

user_address = '0x8c373ed467f3eabefd8633b52f4e1b2df00c9fe8'
already_rated = ['0x006bea43baa3f7a6f765f14f10a1a1b08334ef45','0x5102791ca02fc3595398400bfe0e33d7b6c82267','0x68d57c9a1c35f63e2c83ee8e49a64e9d70528d25','0xc528c28fec0a90c083328bc45f587ee215760a0f']
k = 5

model_dir = '../jobs/wals_ml_local_20190107_235006'

user_map = np.load(model_dir + "/model/user.npy")
item_map = np.load(model_dir + "/model/item.npy")
row_factor = np.load(model_dir + "/model/row.npy")
col_factor = np.load(model_dir + "/model/col.npy")
user_idx = np.searchsorted(user_map, user_address)
user_rated = [np.searchsorted(item_map, i) for i in already_rated]

recommendations = generate_recommendations(user_idx, user_rated, row_factor, col_factor, k)

tokens = [item_map[i] for i in recommendations]

print(tokens)

max_stars_repo_name: BitskiCo/token-recommender

max_issues_repo_name: BitskiCo/token-recommender

max_forks_repo_name: BitskiCo/token-recommender


text:
import numpy as np
from matplotlib import pyplot as plt

def load(name):
    input_images = "./data/" + name
    data = np.load(input_images)  # 28x28 (sound familiar?) grayscale bitmap in numpy .npy format; images are centered
    data = data / 255
    data = np.reshape(data, (data.shape[0], 28, 28, 1))  # fourth dimension is color
    return data

def test_loader(data_set):
    data = load(data_set)
    img_w, img_h = data.shape[1:3]
    print(img_w, img_h)
    print(data.shape)
    plt.imshow(data[8375, :, :, 0], cmap='Greys')
    plt.show()

# test_loader()