text:
"""
send a gRPC command that has streaming results
capture the results in the db as StoredResponse objects
"""
import copy
import uuid
from random import random

import math
import numpy as np

from django.conf import settings
from django.http import JsonResponse

from tworaven_apps.utils.static_keys import KEY_SUCCESS, KEY_DATA

from tworaven_apps.utils.json_helper import json_loads
from tworaven_apps.utils.proto_util import message_to_json
from tworaven_apps.utils.basic_response import (ok_resp, err_resp)
from tworaven_apps.utils.view_helper import get_json_error

from tworaven_apps.ta2_interfaces.ta2_connection import TA2Connection
from tworaven_apps.ta2_interfaces.models import \
    (StoredRequest, StoredResponse)
from tworaven_apps.ta2_interfaces.stored_data_util import StoredRequestUtil

from tworaven_apps.ta2_interfaces.websocket_message import WebsocketMessage
from tworavensproject.celery import celery_app

from tworaven_apps.utils.random_info import get_timestamp_string
from tworaven_apps.utils.file_util import create_directory

import grpc
import core_pb2

from google.protobuf.json_format import \
    (Parse, ParseError)

import shutil
from os import path

import os
import json
from d3m.container.dataset import Dataset

from sklearn.model_selection import train_test_split
import pandas as pd
from pandas.api.types import is_numeric_dtype

#
# Import Tasks to SearchSolutions/GetSearchSolutionsResults,
#                 FitSolution/GetFitSolutionResults,
#                 ScoreSolution/GetScoreSolutionResults
#
from tworaven_apps.ta2_interfaces.ta2_search_solutions_helper import \
    SearchSolutionsHelper
from tworaven_apps.ta2_interfaces.ta2_fit_solution_helper import FitSolutionHelper
from tworaven_apps.ta2_interfaces.ta2_score_solution_helper import ScoreSolutionHelper
from tworaven_apps.user_workspaces.models import UserWorkspace

from tworaven_solver import approx_seconds, get_date
import csv
from dateutil import parser
from collections import deque

@celery_app.task(ignore_result=True)
def stream_and_store_results(raven_json_str, stored_request_id,
                             grpc_req_obj_name, grpc_call_name, **kwargs):
    """Make the grpc call which has a streaming response

    grpc_req_obj_name: "core_pb2.GetSearchSolutionsResultsRequest", etc
    grpc_call_name: "GetSearchSolutionsResults", etc
    """
    core_stub, err_msg = TA2Connection.get_grpc_stub()
    if err_msg:
        StoredRequestUtil.set_error_status(stored_request_id, err_msg)
        return

    # optional: used to stream messages back to client via channels
    #
    websocket_id = kwargs.get('websocket_id', None)

    #
    grpc_req_obj = eval(grpc_req_obj_name)

    grpc_rpc_call_function = eval('core_stub.%s' % grpc_call_name)

    # --------------------------------
    # convert the JSON string to a gRPC request
    #  Yes: done for the 2nd time
    # --------------------------------
    try:
        req = Parse(raven_json_str,
                    grpc_req_obj())
    except ParseError as err_obj:
        err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
        StoredRequestUtil.set_error_status(stored_request_id, err_msg)
        return

    # --------------------------------
    # Send the gRPC request
    # --------------------------------
    msg_cnt = 0
    try:
        # -----------------------------------------
        # Iterate through the streaming responses
        # -----------------------------------------
        for reply in grpc_rpc_call_function(\
                req, timeout=settings.TA2_GRPC_LONG_TIMEOUT):

            msg_cnt += 1

            stored_resp = None  # to hold a StoredResponse object

            # -----------------------------------------
            # parse the response
            # -----------------------------------------
            msg_json_str = message_to_json(reply)

            msg_json_info = json_loads(msg_json_str)

            # -----------------------------------------
            # does it look ok?
            # -----------------------------------------
            if not msg_json_info.success:
                print('PROBLEM HERE TO LOG!')

                user_msg = 'failed to store response: %s' % \
                           msg_json_info.err_msg
                ws_msg = WebsocketMessage.get_fail_message(\
                            grpc_call_name, user_msg, msg_cnt=msg_cnt)
                ws_msg.send_message(websocket_id)

                continue

            # -----------------------------------------
            # Looks good, save the response
            # -----------------------------------------
            stored_resp_info = StoredResponse.add_response(\
                            stored_request_id,
                            response=msg_json_info.result_obj)

            # -----------------------------------------
            # Make sure the response was saved (probably won't happen)
            # -----------------------------------------
            if not stored_resp_info.success:
                # Not good but probably won't happen
                # send a message to the user...
                #
                user_msg = 'failed to store response: %s' % \
                            msg_json_info.err_msg
                ws_msg = WebsocketMessage.get_fail_message(\
                        grpc_call_name, user_msg, msg_cnt=msg_cnt)

                ws_msg.send_message(websocket_id)

                # Wait for the next response...
                continue

            # -----------------------------------------------
            # send responses back to any open WebSockets
            # ---------------------------------------------
            if websocket_id:
                stored_resp = stored_resp_info.result_obj

                ws_msg = WebsocketMessage.get_success_message(\
                                    grpc_call_name,
                                    'it worked',
                                    msg_cnt=msg_cnt,
                                    data=stored_resp.as_dict())

                print('ws_msg: %s' % ws_msg)
                #print('ws_msg', ws_msg.as_dict())

                ws_msg.send_message(websocket_id)

                StoredResponse.mark_as_read(stored_resp)
            # -----------------------------------------------

            print('msg received #%d' % msg_cnt)

    except grpc.RpcError as err_obj:
        StoredRequestUtil.set_error_status(\
                        stored_request_id,
                        str(err_obj))
        return

    #except Exception as err_obj:
    #    StoredRequestUtil.set_error_status(\
    #                    stored_request_id,
    #                    str(err_obj))
    #    return

    StoredRequestUtil.set_finished_ok_status(stored_request_id)

def rewrite_dataset_schema(problem, dataset_schema, all_variables, dataset_id, update_roles=False):
    dataset_schema['about']['datasetID'] = dataset_id

    resource_schema = next(i for i in dataset_schema['dataResources'] if i['resType'] == 'table')

    # rewrite datasetDoc in output datasets if new problem metadata is supplied
    keep_variables = list({
        *problem.get('indexes', ['d3mIndex']),
        *problem['predictors'],
        *problem['targets']
    })

    ordering_column = problem.get('forecastingHorizon', {}).get("column")
    if ordering_column and ordering_column not in keep_variables:
        keep_variables.append(ordering_column)

    # preserve column order, and only keep variables that already existed
    keep_variables = sorted(list(i for i in keep_variables if i in all_variables), key=lambda x: all_variables.index(x))

    passthrough_roles = [
        'multiIndex', 'key', 'interval', 'boundingPolygon',
        'edgeSource', "directedEdgeSource", "undirectedEdgeSource", "multiEdgeSource", "simpleEdgeSource",
        "edgeTarget", "directedEdgeTarget", "undirectedEdgeTarget", "multiEdgeTarget", "simpleEdgeTarget"
    ]

    map_roles = {
        "indexes": 'index',
        'predictors': 'attribute',
        'targets': 'suggestedTarget',
        'crossSection': 'suggestedGroupingKey',
        'location': 'locationIndicator',
        'boundary': 'boundaryIndicator',
        'weights': 'instanceWeight',
        'privileged': 'suggestedPriviligedData'
    }

    def do_update_roles(prev_roles, col_name):
        roles = [i for i in prev_roles if i in passthrough_roles]

        for role_name in map_roles:
            if col_name in problem[role_name]:
                roles.append(map_roles[role_name])
        # if col_name == ordering_column and 'timeIndicator' not in col_schema['role']:
        #     col_schema['role'].append('timeIndicator')
        return roles

    def update_col_schema(i, col_name):
        col_schema = next((col_schema for col_schema in resource_schema['columns'] if col_schema['colName'] == col_name), None)
        if col_schema is None:
            return
        col_schema['colIndex'] = i
        if update_roles:
            col_schema['role'] = do_update_roles(col_schema['role'], col_name)
        return col_schema

    # modify in place
    updated_schemas = []
    for i, col_name in enumerate(keep_variables):
        updated_schema = update_col_schema(i, col_name)
        if updated_schema is None:
            continue
        updated_schemas.append(updated_schema)

    resource_schema['columns'] = updated_schemas

    # these are no longer necessarily valid after rewriting
    resource_schema.pop("columnsCount", None)
    dataset_schema.pop('metadata', None)

    return keep_variables, dataset_schema

@celery_app.task()
def split_dataset(configuration, workspace):

    # parse input data
    split_options = configuration.get('split_options', {})
    problem = configuration.get('problem')
    dataset_schema = json.load(open(configuration['dataset_schema'], 'r'))
    dataset_id = configuration['dataset_id']

    # actual variable names in the data
    all_variables = pd.read_csv(configuration['dataset_path'], nrows=1).columns.tolist()
    # dataset schema will change because predictors and targets does not span all variables
    # dataset id needs to be coordinated or else the TA2 will reject eventual produce calls on other datasets
    keep_variables, dataset_schema = rewrite_dataset_schema(
        problem=problem,
        dataset_schema=dataset_schema,
        all_variables=all_variables,
        dataset_id=dataset_id,
        update_roles=configuration.get('update_roles'))

    # find the learningData resource, which is by convention the first table
    resource_schema = next(i for i in dataset_schema['dataResources'] if i['resType'] == 'table')

    cross_section_date_limits = None
    inferred_freq = None
    dtypes = {}

    # WARNING: dates are assumed to be monotonically increasing
    #    this is to make it possible to support arbitrarily large datasets
    if problem.get('taskType') == 'FORECASTING' and problem['forecastingHorizon']['column']:
        time_column = problem['forecastingHorizon']['column']
        time_format = problem.get('time_format', {}).get(time_column)

        if time_format:
            dtypes[time_column] = str
        for cross_section in problem.get('crossSection', []):
            dtypes[cross_section] = str

        cross_section_date_limits = {}
        time_buffer = []
        candidate_frequencies = set()

        # create generator that provides a data chunk on every iteration
        data_file_generator = pd.read_csv(
            configuration['dataset_path'],
            chunksize=10 ** 5,
            usecols=keep_variables,
            dtype=dtypes)

        infer_count = 0
        for dataframe_chunk in data_file_generator:

            if time_format:
                dataframe_chunk[time_column] = dataframe_chunk[time_column].apply(
                    lambda x: get_date(x, time_format=time_format))
            dataframe_chunk = dataframe_chunk.sort_values(by=[time_column])

            for _, row in dataframe_chunk.iterrows():
        # with open(configuration['dataset_path'], 'r') as infile:
        #     reader = csv.DictReader(infile)
        #     for row in reader:
                date = row[time_column]
                if not date:
                    continue

                # limit the number of times frequency is inferred
                if infer_count < 1000:

                    buffer_is_empty = len(time_buffer) == 0
                    date_is_newer = len(time_buffer) > 0 and time_buffer[-1] < date

                    if buffer_is_empty or date_is_newer:
                        time_buffer.append(date)
                        if len(time_buffer) > 3:
                            del time_buffer[0]

                        # at minimum three time points are needed to infer a date offset frequency
                        if len(time_buffer) == 3:
                            infer_count += 1
                            if time_format:
                                try:
                                    candidate_frequency = pd.infer_freq(time_buffer)

                                    if candidate_frequency:
                                        candidate_frequencies.add(candidate_frequency)
                                except Exception:
                                    # pandas._libs.tslibs.np_datetime.OutOfBoundsDatetime
                                    pass
                            else:
                                candidate_frequencies.add(abs(time_buffer[1] - time_buffer[0]))

                # collect the highest date within each cross section
                section = tuple(row[col] for col in problem.get('crossSection', []))
                cross_section_date_limits.setdefault(section, date)
                cross_section_date_limits[section] = max(cross_section_date_limits[section], date)

        # if data has a trio of evenly spaced records
        if candidate_frequencies:
            if time_format:
                # sort inferred frequency by approximate time durations, select shortest
                inferred_freq = sorted([(i, approx_seconds(i)) for i in candidate_frequencies], key=lambda x: x[1])[0][0]
                inferred_freq = pd.tseries.frequencies.to_offset(inferred_freq)
            else:
                inferred_freq = min(candidate_frequencies)

    # create new directory structure for the data in a role
    def get_dataset_paths(role):
        dest_dir_info = create_destination_directory(workspace, name=role)
        if not dest_dir_info[KEY_SUCCESS]:
            return JsonResponse(get_json_error(dest_dir_info.err_msg))
        dest_directory = dest_dir_info[KEY_DATA]

        # copy all files, remove the csv_path (we'll be overwriting it)
        csv_path = path.join(dest_directory, resource_schema['resPath'])
        shutil.rmtree(dest_directory)
        shutil.copytree(workspace.d3m_config.training_data_root, dest_directory)
        os.remove(csv_path)
        role_dataset_schema_path = path.join(dest_directory, 'datasetDoc.json')

        role_dataset_schema = copy.deepcopy(dataset_schema)

        with open(role_dataset_schema_path, 'w') as dataset_schema_file:
            json.dump(role_dataset_schema, dataset_schema_file)

        return role_dataset_schema_path, role_dataset_schema, csv_path

    all_datasetDoc_path, all_datasetDoc, all_datasetCsv = get_dataset_paths('all')
    dataset_schemas = {'all': all_datasetDoc}
    dataset_schema_paths = {'all': all_datasetDoc_path}
    dataset_paths = {'all': all_datasetCsv}
    # indicates if data was successfully stratified (prone to failure on highly imbalanced data)
    dataset_stratified = {}

    if split_options.get('outOfSampleSplit'):
        train_datasetDoc_path, train_datasetDoc, train_datasetCsv = get_dataset_paths('train')
        test_datasetDoc_path, test_datasetDoc, test_datasetCsv = get_dataset_paths('test')

        dataset_schemas = {
            **dataset_schemas,
            'train': train_datasetDoc,
            'test': test_datasetDoc
        }

        dataset_schema_paths = {
            **dataset_schema_paths,
            'train': train_datasetDoc_path,
            'test': test_datasetDoc_path
        }

        dataset_paths = {
            **dataset_paths,
            'train': train_datasetCsv,
            'test': test_datasetCsv,
        }

        dataset_stratified = {
            'train': split_options.get('stratified'),
            'test': split_options.get('stratified')
        }

    DEFAULT_TRAIN_TEST_RATIO = .7
    train_test_ratio = split_options.get('trainTestRatio', DEFAULT_TRAIN_TEST_RATIO)
    if not (0 < train_test_ratio <= 1):
        raise ValueError("train-test ratio must be between 0 and 1")

    random_seed = split_options.get('randomSeed', 0)

    # traverse the entire file for a row count. Informs the sampling ratios when chunking
    with open(configuration['dataset_path'], 'r') as infile:
        _header_line = next(infile)
        row_count = sum(1 for _ in infile)

    # write out blank csv files for each split
    for split_name in ['train', 'test', 'all']:
        pd.DataFrame(data=[], columns=keep_variables).to_csv(dataset_paths[split_name], index=False, quoting=csv.QUOTE_NONNUMERIC)

    # by default, the split is trivially forever None, which exhausts all zips
    splits_file_generator = iter(lambda: None, 1)
    if split_options.get('splitsDir') and split_options.get('splitsFile'):
        splits_file_path = f"{split_options['splitsDir']}/{split_options['splitsFile']}"
        splits_file_generator = pd.read_csv(splits_file_path, chunksize=10 ** 5)

    data_file_generator = pd.read_csv(
        configuration['dataset_path'],
        chunksize=10 ** 5,
        usecols=keep_variables,
        dtype=dtypes)
    row_count_chunked = 0

    # TODO: adjust chunksize based on number of columns
    for dataframe, dataframe_split in zip(
            data_file_generator,
            splits_file_generator):

        # rows with NaN values become object rows, which may contain multiple types. The NaN values become empty strings
        # this converts '' to np.nan in non-nominal columns, so that nan may be dropped
        # nominals = problem.get('categorical', [])
        # for column in [col for col in problem['targets'] if col not in nominals]:
        #     dataframe[column].replace('', np.nan, inplace=True)

        # drop null values in target variables
        if 'SEMISUPERVISED' not in problem['taskType'] and problem['taskType'] != 'CLUSTERING':
            nominals = problem.get('categorical', [])
            for column in [col for col in problem['targets'] if col not in nominals]:
                dataframe[column].replace('', np.nan, inplace=True)

            dataframe.dropna(inplace=True, subset=problem['targets'])
            dataframe.reset_index(drop=True, inplace=True)

        # max count of each split ['all', 'test', 'train']
        max_count = int(split_options.get('maxRecordCount', 5e4))
        chunk_count = len(dataframe)
        sample_count = int(max_count / row_count * chunk_count)

        if split_options.get('outOfSampleSplit'):
            if dataframe_split:
                train_indices = set(dataframe_split[dataframe_split['type'] == 'TRAIN']['d3mIndex'].tolist())
                test_indices = set(dataframe_split[dataframe_split['type'] == 'TEST']['d3mIndex'].tolist())

                splits = {
                    'train': dataframe[dataframe['d3mIndex'].astype(int).isin(train_indices)],
                    'test': dataframe[dataframe['d3mIndex'].astype(int).isin(test_indices)],
                    'stratified': False
                }

            # split dataset along temporal variable
            elif problem['taskType'] == 'FORECASTING':
                horizon = problem.get('forecastingHorizon', {}).get('value', 10)
                if not horizon:
                    horizon = 10

                if cross_section_date_limits and inferred_freq:
                    time_column = problem['forecastingHorizon']['column']
                    time_format = problem.get('time_format', {}).get(time_column)

                    cross_section_max_count = int(max_count / len(cross_section_date_limits))
                    horizon = min(cross_section_max_count, horizon)

                    def in_test(row):
                        section = tuple(row[col] for col in problem.get('crossSection', []))
                        date = row[time_column]
                        if time_format:
                            date = get_date(date, time_format)

                        if not date:
                            return False
                        max_date = cross_section_date_limits[section]
                        # print('interval:', max_date - inferred_freq * horizon, max_date)
                        return max_date - inferred_freq * horizon <= date <= max_date

                    def in_train(row):
                        section = tuple(row[col] for col in problem.get('crossSection', []))
                        date = row[time_column]
                        if time_format:
                            date = get_date(date, time_format)
                        if not date:
                            return False
                        max_date = cross_section_date_limits[section] - inferred_freq * horizon
                        # TODO: lower bound isn't being set, due to risk of time underflow
                        return date < max_date

                    splits = {
                        'train': dataframe.loc[dataframe.apply(in_train, axis=1)],
                        'test': dataframe.loc[dataframe.apply(in_test, axis=1)],
                        'stratified': False
                    }
                else:
                    train_idx_min = row_count - max_count - horizon
                    test_idx_min = row_count - horizon

                    def clamp(idx):
                        return min(len(dataframe), max(0, int(idx)))

                    splits = {
                        'train': dataframe.iloc[
                                 clamp(train_idx_min - row_count_chunked)
                                 :clamp(test_idx_min - row_count_chunked)],
                        'test': dataframe.iloc[
                                clamp(test_idx_min - row_count_chunked)
                                :clamp(len(dataframe))],
                        'stratified': False
                    }

            else:
                shuffle = split_options.get('shuffle', True)
                stratified = split_options.get('stratified')

                def run_split():
                    try:
                        dataframe_train, dataframe_test = train_test_split(
                            dataframe,
                            train_size=train_test_ratio,
                            shuffle=shuffle,
                            stratified=stratified,
                            random_state=random_seed)
                        return {'train': dataframe_train, 'test': dataframe_test, 'stratified': stratified}
                    except TypeError:
                        dataframe_train, dataframe_test = train_test_split(
                            dataframe,
                            shuffle=shuffle,
                            train_size=train_test_ratio,
                            random_state=random_seed)
                        return {'train': dataframe_train, 'test': dataframe_test, 'stratified': False}

                splits = run_split()

            for split_name in ['train', 'test']:
                if problem['taskType'] != 'FORECASTING':
                    if sample_count < len(splits[split_name]):
                        splits[split_name] = splits[split_name].sample(sample_count)

                splits[split_name].to_csv(
                    dataset_paths[split_name],
                    mode='a', header=False, index=False,
                    quoting=csv.QUOTE_NONNUMERIC)
                dataset_stratified[split_name] = dataset_stratified[split_name] and splits['stratified']

        row_count_chunked += len(dataframe)
        if problem['taskType'] != 'FORECASTING':
            if sample_count < chunk_count:
                dataframe = dataframe.sample(sample_count)

        dataframe.to_csv(dataset_paths['all'], mode='a', header=False, index=False, quoting=csv.QUOTE_NONNUMERIC)

    def get_mode(x):
        mode = pd.Series.mode(x)
        if len(mode):
            return mode[0]
    # aggregate so that each cross section contains one observation at each time point
    ordering_column = problem.get('forecastingHorizon', {}).get('column')
    if problem.get('taskType') == 'FORECASTING' and ordering_column:
        for split_name in dataset_paths:

            # data no longer needs to be chunked, it should be small enough (unless there are a large number of dupe records)
            dataframe = pd.read_csv(
                dataset_paths[split_name],
                dtype=dtypes)

            key_order = dataframe.columns.values

            group_keys = [ordering_column, *problem.get('crossSection', [])]
            other_keys = [i for i in dataframe.columns.values if i not in group_keys]

            grouped = dataframe.groupby(group_keys)
            aggregated = grouped.aggregate(
                {variable: pd.Series.mean if is_numeric_dtype(variable) else get_mode
                 for variable in other_keys})
            aggregated.reset_index(inplace=True)

            # reindex if aggregation shortened the dataframe
            if len(dataframe) != len(aggregated):
                aggregated[problem['indexes'][0]] = range(len(aggregated))

            aggregated = aggregated.reindex(columns=key_order, copy=False)
            aggregated.to_csv(dataset_paths[split_name], index=False, quoting=csv.QUOTE_NONNUMERIC)

    return {
        'dataset_schemas': dataset_schemas,
        'dataset_schema_paths': dataset_schema_paths,
        'dataset_paths': dataset_paths,
        'stratified': dataset_stratified
    }

def create_destination_directory(user_workspace, name):
    """Used to add a write directory for the partials app"""
    if not isinstance(user_workspace, UserWorkspace):
        return err_resp('Error "user_workspace" must be a UserWorkspace object.')

    # build destination path for partials app
    dest_dir_path = os.path.join(user_workspace.d3m_config.get_temp_directory(),
                         f'ws_{user_workspace.id}',
                         name,
                         get_timestamp_string())

    new_dir_info = create_directory(dest_dir_path)
    if not new_dir_info.success:
        return {KEY_SUCCESS: False, KEY_DATA: f' {new_dir_info.err_msg} ({dest_dir_path})'}

    return {KEY_SUCCESS: True, KEY_DATA: dest_dir_path}

@celery_app.task
def create_partials_datasets(configuration, workspace_id):
    """Create partials datasets"""
    # print('configuration', configuration)

    problem = configuration['problem']
    dataset_id = configuration['dataset_id']

    try:
        workspace = UserWorkspace.objects.get(pk=workspace_id)
    except UserWorkspace.DoesNotExist:
        return {
            KEY_SUCCESS: False,
            KEY_DATA: f' UserWorkspace not found for id {workspace_id}.'
        }

    MAX_DATASET_SIZE = 50
    MAX_DOMAIN_SIZE = 100
    # load dataframe and dataset schema
    if 'dataset_schema_path' in configuration:
        dataset_schema = json.load(open(configuration['dataset_schema_path'], 'r'))
        print(f'dataset_schema_path: file://{configuration["dataset_schema_path"]}')
        dataset = Dataset.load(f'file://{configuration["dataset_schema_path"]}')
    elif 'dataset' in configuration:
        dataset_schema = configuration['dataset_schema']
        dataframe = pd.DataFrame(configuration['dataset'])
    else:
        return {KEY_SUCCESS: False, KEY_DATA: 'no dataset supplied'}

    keep_variables, dataset_schema = rewrite_dataset_schema(
        problem=problem,
        dataset_schema=dataset_schema,
        all_variables=configuration['all_variables'],
        dataset_id=dataset_id,
        update_roles=configuration.get('update_roles'))

    #print(dataset_schema)
    resource_schema = next(i for i in dataset_schema['dataResources'] if i['resType'] == 'table')

    if 'dataset_schema_path' in configuration:
        dataframe = dataset[resource_schema['resID']]

    domains = {k: v for k, v in configuration['domains'].items() if k in keep_variables}
    dataframe = dataframe[keep_variables]
    # METADATA OF SCHEMA:
    # {variable: [domain], ...}

    if len(dataframe) > MAX_DATASET_SIZE:
        return {KEY_SUCCESS: False, KEY_DATA: 'initial dataset too large to expand into partials'}

    def write_dataset(name, writable_dataframe):
        dest_dir_info = create_destination_directory(workspace, name=name)
        if not dest_dir_info[KEY_SUCCESS]:
            return dest_dir_info

        dest_directory = dest_dir_info[KEY_DATA]
        csv_path = os.path.join(dest_directory, resource_schema['resPath'])
        shutil.rmtree(dest_directory)
        shutil.copytree(workspace.d3m_config.training_data_root, dest_directory)

        dataset_schema_path = os.path.join(dest_directory, 'datasetDoc.json')

        # D3M: change the dataset ID to be consistent with the search
        with open(dataset_schema_path, 'w') as dataset_schema_file:
            json.dump(dataset_schema, dataset_schema_file)

        os.remove(csv_path)
        writable_dataframe.to_csv(csv_path, index=False, quoting=csv.QUOTE_NONNUMERIC)

        return {KEY_SUCCESS: True, KEY_DATA: (path.join(dest_directory, 'datasetDoc.json'), csv_path)}

    dataset_index_paths = {}
    dataset_schemas = {}
    dataset_paths = {}

    # rename d3mIndex to d3mIndexOriginal
    new_column_names = list(dataframe.columns.values)
    if 'd3mIndex' in new_column_names:
        d3mIndexIndex = new_column_names.index('d3mIndex')
        new_column_names[d3mIndexIndex] = str(new_column_names[d3mIndexIndex]) + 'Original'

    union_datasets = []
    for predictor in domains:
        synthetic_data = []
        predictor_idx = new_column_names.index(predictor)
        for row_idx in range(len(dataframe)):
            row = dataframe.iloc[row_idx].tolist()

            for support_member in domains[predictor][:MAX_DOMAIN_SIZE]:
                row_copy = list(row)
                row_copy[predictor_idx] = support_member
                synthetic_data.append(row_copy)

        synthetic_data = pd.DataFrame(synthetic_data, columns=new_column_names)

        if configuration['separate_variables']:
            synthetic_data.insert(0, 'd3mIndex', list(range(len(synthetic_data))))
            dataset_name = configuration['name'] + predictor

            dataset_index_name = dataset_name + '_index'
            result_index_write = write_dataset(dataset_index_name, synthetic_data[['d3mIndex', 'd3mIndexOriginal']])
            if not result_index_write[KEY_SUCCESS]:
                return result_index_write
            dataset_index_paths[dataset_name] = result_index_write[KEY_DATA][1]

            # the existence of this column in the original data breaks TA2s
            synthetic_data.drop(columns=['d3mIndexOriginal'], inplace=True)

            result_write = write_dataset(dataset_name, synthetic_data)
            if not result_write[KEY_SUCCESS]:
                return result_write
            dataset_schema_variable, dataset_path_variable = result_write[KEY_DATA]

            dataset_schemas[dataset_name] = dataset_schema_variable
            dataset_paths[dataset_name] = dataset_path_variable
        else:
            union_datasets.append(synthetic_data)

    if union_datasets:
        synthetic_data = pd.concat(union_datasets)
        synthetic_data.insert(0, 'd3mIndex', list(range(len(synthetic_data))))

        dataset_name = configuration['name']
        dataset_index_name = dataset_name + '_index'

        result_index_write = write_dataset(dataset_index_name, synthetic_data[['d3mIndex', 'd3mIndexOriginal']])
        if not result_index_write[KEY_SUCCESS]:
            return result_index_write
        dataset_index_paths[dataset_name] = result_index_write[KEY_DATA][1]

        # the existence of this column in the original data breaks TA2s
        synthetic_data.drop(columns=['d3mIndexOriginal'], inplace=True)

        result_write = write_dataset(dataset_name, synthetic_data)
        if not result_write[KEY_SUCCESS]:
            return result_write
        dataset_schema_union, dataset_path_union = result_write[KEY_DATA]

        dataset_schemas[configuration['name']] = dataset_schema_union
        dataset_paths[configuration['name']] = dataset_path_union

    return {
        KEY_SUCCESS: True,
        KEY_DATA: {
            'dataset_schemas': dataset_schemas,
            'dataset_paths': dataset_paths,
            'dataset_index_paths': dataset_index_paths
        }
    }

max_stars_repo_name: TwoRavens/TwoRavens

max_issues_repo_name: TwoRavens/TwoRavens

max_forks_repo_name: TwoRavens/TwoRavens


text:
import numpy as np
from sklearn.feature_selection import mutual_info_classif
from ReliefF import ReliefF
import FCBF
import su_calculation as su
import MRMR as mr
import JMI

def infogain(X, y, n_features):
    '''
    Runs infogain feature selection on the data (X) and the target values (y) and finds
    the index of the top n_features number of features.

    Args:
        X: A Numpy array containing the dataset
        y: A Numpy array consisting of the target values
        n_features: An integer specifying how many features should be selected

    Returns a list containing the indices of the features that have been selected
    '''
    score = mutual_info_classif(X, y,random_state=0)
    index = list(np.argsort(list(score))[-1*n_features:])
    index.sort()
    return index

def reliefF(X, y, n_features):
    '''
    Runs ReliefF algorithm on the data (X) and the target values (y) and finds
    the index of the top n_features number of features.

    Args:
        X: A Numpy array containing the dataset
        y: A Numpy array consisting of the target values
        n_features: An integer specifying how many features should be selected

    Returns a list containing the indices of the features that have been selected
    '''
    fs = ReliefF(n_neighbors=100, n_features_to_keep=n_features)
    fs.fit_transform(X, y)
    index = fs.top_features[:n_features]
    return index

def fcbf(X, y):
    '''
    Runs Fast Correlation-Based Filter feature selection on the data (X) and the target values (y) and finds
    the index of the significant features.

    Args:
        X: A Numpy array containing the dataset
        y: A Numpy array consisting of the target values
        n_features: An integer specifying how many features should be selected

    Returns a list containing the indices of the features that have been selected
    '''
    selection = FCBF.fcbf(X, y)
    index = list(selection[0])
    index.sort()
    return index

def merit_calculation(X, y):
    '''
    This function calculates the merit of X given class labels y, where
    merits = (k * rcf)/sqrt(k+k*(k-1)*rff)
    rcf = (1/k)*sum(su(fi,y)) for all fi in X
    rff = (1/(k*(k-1)))*sum(su(fi,fj)) for all fi and fj in X

    Args:
        X: A Numpy array containing the dataset
        y: A Numpy array consisting of the target values

    Returns: Merits (float) of a feature subset X
    '''
    n_samples, n_features = X.shape
    rff = 0
    rcf = 0
    for i in range(n_features):
        fi = X[:, i]
        rcf += su.su_calculation(fi, y)
        for j in range(n_features):
            if j > i:
                fj = X[:, j]
                rff += su.su_calculation(fi, fj)
    rff *= 2
    merits = rcf / np.sqrt(n_features + rff)
    return merits

def cfs(X, y):
    '''
    Runs Correlation-based feature selection on the data (X) and the target values (y) and finds
    the index of the significant features.

    Args:
        X: A Numpy array containing the dataset
        y: A Numpy array consisting of the target values
        n_features: An integer specifying how many features should be selected

    Returns a list containing the indices of the features that have been selected
    '''
    n_samples, n_features = X.shape
    F = []

    M = []
    while True:
        merit = -100000000000
        idx = -1
        for i in range(n_features):
            if i not in F:
                F.append(i)
                t = merit_calculation(X[:, F], y)
                if t > merit:
                    merit = t
                    idx = i
                F.pop()
        F.append(idx)
        M.append(merit)
        if len(M) > 6:
            if M[len(M)-1] <= M[len(M)-2]:
                if M[len(M)-2] <= M[len(M)-3]:
                    if M[len(M)-3] <= M[len(M)-4]:
                        if M[len(M)-4] <= M[len(M)-5]:
                            break
    return np.array(F)

def run_feature_selection(method,X,y):
    '''
    Runs the specific ranking based feature selection method.

    Args:
        method: A string that refers to the feature selection method to be used
        X: An Arraylike structure containing the dataset
        y: An Arraylike structure consisting of the target values

    Returns:
        A list containing the indices of the features that have been selected
    '''
    X=np.array(X)
    y=np.array(y)
    if method[:3] == 'cfs':
        return cfs(X, y)
    elif method[:3] == 'jmi':
        if(len(method.split("_"))==1):
            return JMI.jmi(X,y)
        else:
            return JMI.jmi(X,y, n_selected_features = int(method.split("_")[1]))
    elif method[:4] == 'mrmr':
        if(len(method.split("_"))==1):
            return mr.mrmr(X,y)
        else:
            return mr.mrmr(X,y, n_selected_features = int(method.split("_")[1]))
    elif method[:4] =='fcbf':
        return fcbf(X,y)
    elif method[:7] == 'reliefF':
        return reliefF(X,y,int(method[8:]))
    elif method[:8] == 'infogain':
        return infogain(X,y,int(method[9:]))

ext: py

max_stars_repo_name: aziz-zafar/Machine-Learning-Package

max_issues_repo_name: aziz-zafar/Machine-Learning-Package

max_forks_repo_name: aziz-zafar/Machine-Learning-Package

reason: import numpy


text:
"""Authors: Cody Baker, Ben Dichter and Luiz Tauffer."""
from datetime import datetime
import pkgutil
import importlib
from pathlib import Path
import typing
import inspect

import numpy as np
import pynwb

from .json_schema_utils import get_base_schema

def get_schema_from_hdmf_class(hdmf_class):
    """Get metadata schema from hdmf class."""
    schema = get_base_schema()
    schema['tag'] = hdmf_class.__module__ + '.' + hdmf_class.__name__

    # Detect child-like (as opposed to link) fields
    pynwb_children_fields = [f['name'] for f in hdmf_class.get_fields_conf() if f.get('child', False)]
    # For MultiContainerInterface
    if hasattr(hdmf_class, '__clsconf__'):
        pynwb_children_fields.append(hdmf_class.__clsconf__['attr'])

    # Temporary solution before this is solved: https://github.com/hdmf-dev/hdmf/issues/475
    if 'device' in pynwb_children_fields:
        pynwb_children_fields.remove('device')

    docval = hdmf_class.__init__.__docval__
    for docval_arg in docval['args']:
        schema_arg = {docval_arg['name']: dict(description=docval_arg['doc'])}

        # type float
        if docval_arg['type'] == 'float' \
            or (isinstance(docval_arg['type'], tuple)
                and any([it in docval_arg['type'] for it in [float, 'float']])):
            schema_arg[docval_arg['name']].update(type='number')

        # type string
        elif docval_arg['type'] is str \
            or (isinstance(docval_arg['type'], tuple)
                and str in docval_arg['type']):
            schema_arg[docval_arg['name']].update(type='string')

        # type datetime
        elif docval_arg['type'] is datetime \
            or (isinstance(docval_arg['type'], tuple)
                and datetime in docval_arg['type']):
            schema_arg[docval_arg['name']].update(type='string', format='date-time')

        # if TimeSeries, skip it
        elif docval_arg['type'] is pynwb.base.TimeSeries \
            or (isinstance(docval_arg['type'], tuple)
                and pynwb.base.TimeSeries in docval_arg['type']):
            continue

        # if PlaneSegmentation, skip it
        elif docval_arg['type'] is pynwb.ophys.PlaneSegmentation or \
                (isinstance(docval_arg['type'], tuple) and
                 pynwb.ophys.PlaneSegmentation in docval_arg['type']):
            continue

        else:
            if not isinstance(docval_arg['type'], tuple):
                docval_arg_type = [docval_arg['type']]
            else:
                docval_arg_type = docval_arg['type']

            # if another nwb object (or list of nwb objects)
            if any([hasattr(t, '__nwbfields__') for t in docval_arg_type]):
                is_nwb = [hasattr(t, '__nwbfields__') for t in docval_arg_type]
                item = docval_arg_type[np.where(is_nwb)[0][0]]
                # if it is child
                if docval_arg['name'] in pynwb_children_fields:
                    items = [get_schema_from_hdmf_class(item)]
                    schema_arg[docval_arg['name']].update(
                        type='array', items=items, minItems=1, maxItems=1
                    )
                # if it is link
                else:
                    target = item.__module__ + '.' + item.__name__
                    schema_arg[docval_arg['name']].update(
                        type='string',
                        target=target
                    )
            else:
                continue

        # Check for default arguments
        if 'default' in docval_arg:
            if docval_arg['default'] is not None:
                schema_arg[docval_arg['name']].update(default=docval_arg['default'])
        else:
            schema['required'].append(docval_arg['name'])

        schema['properties'].update(schema_arg)

    if 'allow_extra' in docval:
        schema['additionalProperties'] = docval['allow_extra']

    return schema

def get_schema_for_NWBFile():
    schema = get_base_schema()
    schema['tag'] = 'pynwb.file.NWBFile'
    schema['required'] = ["session_description", "identifier", "session_start_time"]
    schema['properties'] = {
        "session_description": {
            "type": "string",
            "format": "long",
            "description": "a description of the session where this data was generated"
        },
        "identifier": {
            "type": "string",
            "description": "a unique text identifier for the file"
        },
        "session_start_time": {
            "type": "string",
            "description": "the start date and time of the recording session",
            "format": "date-time"
        },
        "experimenter": {
            "type": "array",
            "items": {"type": "string", "title": "experimenter"},
            "description": "name of person who performed experiment"
        },
        "experiment_description": {
            "type": "string",
            "description": "general description of the experiment"
        },
        "session_id": {
            "type": "string",
            "description": "lab-specific ID for the session"
        },
        "institution": {
            "type": "string",
            "description": "institution(s) where experiment is performed"
        },
        "notes": {
            "type": "string",
            "description": "Notes about the experiment."
        },
        "pharmacology": {
            "type": "string",
            "description": "Description of drugs used, including how and when they were administered. Anesthesia(s), "
                           "painkiller(s), etc., plus dosage, concentration, etc."
        },
        "protocol": {
            "type": "string",
            "description": "Experimental protocol, if applicable. E.g., include IACUC protocol"
        },
        "related_publications": {
            "type": "string",
            "description": "Publication information.PMID, DOI, URL, etc. If multiple, concatenate together and describe"
                           " which is which. such as PMID, DOI, URL, etc"
        },
        "slices": {
            "type": "string",
            "description": "Description of slices, including information about preparation thickness, orientation, "
                           "temperature and bath solution"
        },
        "source_script": {
            "type": "string",
            "description": "Script file used to create this NWB file."
        },
        "source_script_file_name": {
            "type": "string",
            "description": "Name of the source_script file"
        },
        "data_collection": {
            "type": "string",
            "description": "Notes about data collection and analysis."
        },
        "surgery": {
            "type": "string",
            "description": "Narrative description about surgery/surgeries, including date(s) and who performed surgery."
        },
        "virus": {
            "type": "string",
            "description": "Information about virus(es) used in experiments, including virus ID, source, date made, "
                           "injection location, volume, etc."
        },
        "stimulus_notes": {
            "type": "string",
            "description": "Notes about stimuli, such as how and where presented."
        },
        "lab": {
            "type": "string",
            "description": "lab where experiment was performed"
        }
    }
    return schema

def _recurse_subclasses(cls, leaves_only=True) -> list:
    """
    Given some class, find its subclasses recursively

    See: https://stackoverflow.com/a/17246726/13113166

    Args:
        leave_only (bool): If True, only include classes that have no further subclasses,
        if False, return all subclasses.

    Returns:
        list of subclasses
    """

    all_subclasses = []

    for subclass in cls.__subclasses__():
        if leaves_only:
            if len(subclass.__subclasses__()) == 0:
                all_subclasses.append(subclass)
        else:
            all_subclasses.append(subclass)
        all_subclasses.extend(_recurse_subclasses(subclass))

    return all_subclasses

def _recursive_import(module_name:str) -> typing.List[str]:
    """
    Given some path in a python package, import all modules beneath it

    Args:
        module_name (str): name of module to recursively import

    Returns:
        list of all modules that were imported
    """

    # iterate through modules, importing
    # see https://codereview.stackexchange.com/a/70282

    # import module (shouldnt hurt if it has already)
    base_mod = importlib.import_module(module_name)

    pkg_dir = Path(inspect.getsourcefile(base_mod)).resolve().parent

    loaded_modules = []
    for (module_loader, name, ispkg) in pkgutil.walk_packages([pkg_dir], base_mod.__package__+'.'):
        if not ispkg:
            importlib.import_module(name)
            loaded_modules.append(name)

    return loaded_modules

max_stars_repo_name: sneakers-the-rat/nwb-conversion-tools

max_issues_repo_name: sneakers-the-rat/nwb-conversion-tools

max_forks_repo_name: sneakers-the-rat/nwb-conversion-tools


text:
# coding:utf-8
from __future__ import print_function
import numpy as np
import scipy as sp
import math
import copy

class Laminate(object):
	"""docstring for Laminite_"""
	layer_counter = 0
	layer_name_list = []

	def __init__(self,lamina_num = 0,name = 'Layer_' , degradation = 0.001):
		super(Laminate, self).__init__()
		self.lamina_list = []
		self.lamina_num = len(self.lamina_list)
		self.THICK= 0.0
		self.Qk = []   # collect the Q_bar from signal lamina
		self.T_Stress =[]
		self.T_Strain =[]
		self.Zk = []
		self.density = 0
		# self.Zk.append(0.0)
		#hidden huge bug !!! important !!! read the formula
		#add in update()
		self.updated = 0
		self.degradation = degradation

		self.A = np.matrix([[0.0,0.0,0.0],[0.0,0.0,0.0],[0.0,0.0,0.0]])

		self.B = np.matrix([[0.0,0.0,0.0],[0.0,0.0,0.0],[0.0,0.0,0.0]])

		self.D = np.matrix([[0.0,0.0,0.0],[0.0,0.0,0.0],[0.0,0.0,0.0]])

##################################################
# Rule of Mixture is a weighted mean approach, provides a quick
# estimation of composite elastic properties.
# • Assumptions
# 1. Fibers are uniformly distributed throughout the matrix.
# 2. Perfect bonding between fibers and matrix.
# 3. Matrix is free of voids.
# 4. Applied loads are either parallel or normal to the fiber direction.
# 5. No residual stresses.
# 6. Fiber and matrix are linearly elastic materials.
##################################################
	def Rule_of_Mixture(self):
		pass

	def update(self):
		if True: #self.updated == 0:
			self.THICK= 0.0
			self.Qk = []   # collect the Q_bar from signal lamina
			self.T_Stress =[]
			self.T_Strain =[]
			self.Zk = []
			self.Zk.append(0.0)     #hidden huge bug !!! important !!! read the formula
			self.lamina_num = len(self.lamina_list)
			self.density = 0

		self.A = np.matrix([[0.0,0.0,0.0],[0.0,0.0,0.0],[0.0,0.0,0.0]])

		self.B = np.matrix([[0.0,0.0,0.0],[0.0,0.0,0.0],[0.0,0.0,0.0]])

		self.D = np.matrix([[0.0,0.0,0.0],[0.0,0.0,0.0],[0.0,0.0,0.0]])

		for lamina in self.lamina_list:

			lamina.properities_degradation( self.degradation )
			self.Qk.append(lamina.matrix_Q_bar)
			self.T_Stress.append(lamina.matrix_Tstress)
			self.T_Strain.append(lamina.matrix_Tstrain)
			self.THICK = self.THICK + lamina.thickness * 1.0
			self.Zk.append(self.THICK)# self.Zk.reverse()
			self.density = self.density + lamina.density * lamina.thickness * 1.0

		self.density = self.density/self.THICK
#************************************************************************
#	Get A, B, D Matrix
#   Zk[k] - 1/2 laminate's thickness get the coordinate value of Zk in formula
#	Zk[] is the distance between middle line
#************************************************************************
		for i in range(0,3):
			for j in range(0,3):
				for k in range(0,self.lamina_num):
					qtemp = self.Qk[k]
					self.A[i, j] = self.A[i, j] + (self.Zk[k + 1] - self.Zk [k]) * qtemp[i, j]
					self.B[i, j] = self.B[i, j] + ((self.Zk[k + 1] - (self.THICK)/2) ** 2 - (self.Zk[k] - (self.THICK)/2) ** 2) * qtemp[i, j] / 2.0
					self.D[i, j] = self.D[i, j] + ((self.Zk[k + 1] - (self.THICK)/2) ** 3 - (self.Zk[k] - (self.THICK)/2) ** 3) * qtemp[i, j] / 3.0

#************************************************************************
#	**get Astar, Bstar, Cstar, Dstar Matrix
#  ------------------------
#   strain0    A* | B*  N
#   ------ = -------- ---
#	  M        C* | D*  K
#  ------------------------
#************************************************************************
		self.Astar = (self.A).I
		self.Bstar = -1.0*self.Astar * self.B
		self.Cstar = self.B * self.Astar
		self.Dstar = self.D - self.B * self.Astar * self.B

#************************************************************************
#	**get Aprime, Bprime, Cprime, Dprime Matrix
#  ------------------------
#   strain0    A' | B'  N
#   ------- = -------- ---
#	   K       C' | D'  M
#  ------------------------
#************************************************************************
		self.Dprime = (self.Dstar).I
		self.Aprime = self.Astar - self.Bstar * self.Dprime * self.Cstar
		self.Bprime = self.Bstar * self.Dprime
		self.Cprime = self.Bprime.T

		self.ABD = np.bmat([[self.A,self.B],[self.B,self.D]])
		self.abd = (self.ABD).I
		self.detABD = np.linalg.det(self.ABD)
############################################################################
# get the effective  Young's moduli
############################################################################
		Axx = np.matrix([[self.A[1,1],self.A[1,2]],\
						[self.A[1,2],self.A[2,2]]])

		Bxx = np.matrix([[self.B[0,1], self.B[1,1], self.B[1,2]],\
						[self.B[0,2], self.B[1,2], self.B[2,2]]])

		effective_Exx = np.bmat([[Axx,Bxx],[Bxx.T,self.D]])

		self.Ex = self.detABD/(np.linalg.det(effective_Exx) * self.THICK)

#--------------------------------------------------------------------------------

		Axx = np.matrix([[self.A[0,0], self.A[0,2]],\
						[self.A[0,2], self.A[2,2]]])

		Bxx = np.matrix([[self.B[0,0], self.B[0,1], self.B[0,2]],\
						[self.B[0,2], self.B[1,2], self.B[2,2]]])

		effective_Eyy = np.bmat([[Axx,Bxx],[Bxx.T,self.D]])

		self.Ey = self.detABD/(np.linalg.det(effective_Eyy) * self.THICK)

#--------------------------------------------------------------------------------

		# Axx = np.matrix([[self.A[0,0], self.A[0,1]],\
		# 				[self.A[0,1], self.A[1,1]]])

		# Bxx = np.matrix([[self.B[0,0], self.B[0,1], self.B[0,2]],\
		# 				[self.B[0,1], self.B[1,1], self.B[1,2]]])

		# effective_Gxy = np.bmat([[Axx,Bxx],[Bxx.T,self.D]])
		effective_Gxy=\
		np.matrix([
				[self.A[0,0], self.A[0,1], self.B[0,0], self.B[0,1], self.B[0,2]],
				[self.A[0,1], self.A[1,1], self.B[0,1], self.B[1,1], self.B[1,2]],
				[self.B[0,0], self.B[0,1], self.D[0,0], self.D[0,1], self.D[0,2]],
				[self.B[0,1], self.B[1,1], self.D[0,1], self.D[1,1], self.D[1,2]],
				[self.B[0,2], self.B[1,2], self.D[0,2], self.D[1,2], self.D[2,2]]
				])

		self.Gxy = self.detABD/(np.linalg.det(effective_Gxy) * self.THICK)
		self.G = self.Gxy
#--------------------------------------------------------------------------------
# 	  - Epison_y
#  Vxy = ------------
#	   Epison_x
#--------------------------------------------------------------------------------
		Axx = np.matrix([[self.A[0,1], self.A[1,2]],\
						[self.A[0,2], self.A[2,2]]])

		Bxx = np.matrix([[self.B[0,1], self.B[1,1], self.B[1,2]],\
						[self.B[0,2], self.B[1,2], self.B[2,2]]])

		Bxx2 = np.matrix([[self.B[0,0], self.B[0,1], self.B[0,2]],\
						[self.B[0,2], self.B[1,2], self.B[2,2]]])

		effective_Vxy = np.bmat([[Axx,Bxx],[Bxx2.T,self.D]])

		self.Vxy = -1 * np.linalg.det(effective_Vxy) / np.linalg.det(effective_Exx)

#--------------------------------------------------------------------------------
# 		  - Epison_y
#  Vyx = ------------
#		   Epison_x
#--------------------------------------------------------------------------------
		Axx = np.matrix([[self.A[0,1], self.A[1,2]],\
						[self.A[0,2], self.A[2,2]]])

		Bxx = np.matrix([[self.B[0,0], self.B[0,1], self.B[0,2]],\
						[self.B[0,2], self.B[1,2], self.B[2,2]]])

		effective_Vyx = np.bmat([[Axx,Bxx],[Bxx.T,self.D]])

		Axxy = np.matrix([[self.A[0,0], self.A[0,2]],\
						[self.A[0,2], self.A[2,2]]])

		Bxxy = np.matrix([[self.B[0,1], self.B[1,1], self.B[1,2]],\
						[self.B[0,2], self.B[1,2], self.B[2,2]]])

		effective_Vyxy = np.bmat([[Axxy,Bxxy],[Bxxy.T,self.D]])

		self.Vyx = -1 * np.linalg.det(effective_Vyx) / np.linalg.det(effective_Vyxy)

#--------------------------------------------------------------------------------

		self.updated = 1

	def add_Lamina(self,lamina):
		# self.lamina_list.append(lamina)
		self.lamina_list.append(copy.deepcopy(lamina))
		self.updated = 0

	def repalce_Lamina(self,number,lamina):
		del self.lamina_list[number]
		self.lamina_list.insert(number, lamina)
		self.updated = 0

	def remove_Lamina(self,number):
		del self.lamina_list[number]
		self.updated = 0

max_stars_repo_name: Eacaen/CLT-material-properties

max_stars_repo_licenses: FTL CNRI-Python Xnet X11

max_issues_repo_name: Eacaen/CLT-material-properties

max_issues_repo_licenses: FTL CNRI-Python Xnet X11

max_forks_repo_name: Eacaen/CLT-material-properties

max_forks_repo_licenses: FTL CNRI-Python Xnet X11

reason: import numpy,import scipy


text:
from datetime import datetime
from functools import partial
import numpy as np
import utils
from pagerank import PageRank, MCPageRank
from hits import HITS, MCHITS
import matplotlib.pyplot as plt
import seaborn as sns
sns.set(context='poster', style='whitegrid')

if __name__ == '__main__':
    TELEPORTING_PROB = 0.15
    NUM_ITER = 4
    EXP_COUNT = 1
    SPARSITY = 0.01
    MAX_SIZE = 10
    MIN_SIZE = 6

    mcpr = MCPageRank(TELEPORTING_PROB)
    pr = PageRank(TELEPORTING_PROB)
    hits = HITS()
    mchits = MCHITS(1 - TELEPORTING_PROB)

    alg_dict = {
        'PageRank': partial(pr.calculate, verbose=True),
        'PI PageRank': partial(pr.iterative_calculate, verbose=True,
                               iter_count=NUM_ITER * 10),
        'MCPageRank': partial(mcpr.calculate, verbose=True, num_iter=NUM_ITER),
        'HITS': partial(hits.calculate, verbose=True),
        'PI HITS': partial(hits.iterative_calculate, num_iter=NUM_ITER * 10,
                           verbose=True),
        'MCHITS': partial(mchits.calculate, verbose=True, num_iter=NUM_ITER)}

    alg_results = {}
    for a, f in alg_dict.items():
        print('Benchmarking', a)
        if a not in alg_results:
            alg_results[a] = []
        for _ in range(EXP_COUNT):
            for i in range(MIN_SIZE, MAX_SIZE + 1):
                print('\tfor size', 2 ** i)
                web = utils.random_graph(2 ** i, SPARSITY)
                _, delta = f(web)
                alg_results[a].append((i, delta.total_seconds()))

    fig = plt.figure(figsize=(18, 10), dpi=150)
    ax = fig.add_subplot(111)
    for a, r in alg_results.items():
        ax.plot(
            list(map(lambda x: x[0], r)),
            list(map(lambda x: np.log10(1 + x[1]), r)),
            alpha=0.75
        )
        ax.scatter(
            list(map(lambda x: x[0], r)),
            list(map(lambda x: np.log10(1 + x[1]), r)),
            label=a,
            alpha=0.8,
            edgecolors='black'
        )
    ax.set_title('Run-time comparison')
    ax.set_ylabel('$\\log_{10}(1 + t)$ s - Time')
    ax.set_xlabel('$\\log_2(n)$ - Number of nodes')
    ax.set_xticks(list(range(MIN_SIZE, MAX_SIZE + 1)))
    ax.set_xlim([MIN_SIZE - 0.5, 0.5 + MAX_SIZE])
    fig.legend(loc='lower center', ncol=6)
    plt.savefig('figures/run-time.png', bbox_inches='tight')


text:
from sympy import Matrix, S, latex
from sympy.abc import x, y, z
from lmi_sdp.lm import lm_sym_expanded
from lmi_sdp.lmi import ShapeError, NonSymmetricMatrixError, LMI_PSD, \
    LMI_NSD, LMI_PD, LMI_ND, init_lmi_latex_printing

def test_LMI_PSD():
    m = Matrix([[x, y], [y, z+1]])
    lmi = LMI_PSD(m)
    assert lmi.lhs == lmi.gts == m
    assert lmi.rhs == lmi.lts == S(0)

    c = Matrix([[0, 1], [1, 2]])
    lmi = LMI_PSD(m, c)
    assert lmi.lhs == lmi.gts == m
    assert lmi.rhs == lmi.lts == c

def test_LMI_canonical():
    m = Matrix([[x, y], [y, z+1]])
    c = Matrix([[0, 1], [1, 2]])
    can_lhs = m-c
    can_rhs = 0

    can = LMI_PSD(m, c).canonical()
    assert can.lhs == can.gts == can_lhs
    assert can.rhs == can.lts == can_rhs

    can = LMI_PSD(0, c-m).canonical()
    assert can.lhs == can.gts == can_lhs
    assert can.rhs == can.lts == can_rhs

    can = LMI_PD(m, c).canonical()
    assert can.lhs == can.gts == can_lhs
    assert can.rhs == can.lts == can_rhs

    can = LMI_NSD(c, m).canonical()
    assert can.lhs == can.gts == can_lhs
    assert can.rhs == can.lts == can_rhs
    assert isinstance(can, LMI_PSD)

    can = LMI_ND(c, m).canonical()
    assert can.lhs == can.gts == can_lhs
    assert can.rhs == can.lts == can_rhs
    assert isinstance(can, LMI_PD)

def test_LMI_canonical_same():
    m = Matrix([[x, y], [y, z+1]])

    lmi = LMI_PSD(m)
    assert lmi is lmi.canonical()

    lmi = LMI_PD(m)
    assert lmi is lmi.canonical()

def test_LMI_expanded():
    variables = (x, y, z)
    m = Matrix([[x, y], [y, z+1]])
    c = Matrix([[0, 1], [1, 2]])

    expdd = LMI_ND(m+c, c).expanded(variables)
    assert expdd == LMI_ND(lm_sym_expanded(m+c, variables), c)

    expdd = LMI_PSD(m).expanded(variables)
    assert expdd == LMI_PSD(lm_sym_expanded(m, variables))

def test_LMI_PSD_exceptions():
    try:
        LMI_PSD(Matrix([[1, x], [y, z]]))
    except NonSymmetricMatrixError:
        pass
    else:
        assert False

    try:
        LMI_PSD(Matrix([[1, x], [y, z]]), assert_symmetry=False)
    except NonSymmetricMatrixError:
        assert False

    try:
        LMI_PSD(Matrix([[x+y]]), Matrix([[x, y], [y, z+1]]))
    except ShapeError:
        pass
    else:
        assert False

def test_lmi_latex_printing():

    init_lmi_latex_printing()

    lmi = LMI_PSD(Matrix([[x, y], [y, z+1]]))

    assert r"\succeq 0" == latex(lmi)[-9:]

max_stars_repo_name: rhaschke/PyLMI-SDP

max_issues_repo_name: rhaschke/PyLMI-SDP

max_forks_repo_name: rhaschke/PyLMI-SDP

reason: from sympy


text:
import numpy as np
import unittest as ut

from scipy.stats import unitary_group

from qfactor.gates import Gate

class TestGetTensorFormat ( ut.TestCase ):

    def test_get_tensor_format ( self ):
        utry = unitary_group.rvs( 8 )
        gate = Gate( utry, (0, 1, 2) )
        tensor = gate.get_tensor_format()
        self.assertTrue( len( tensor.shape ) == 6 )
        self.assertTrue( all( [ x == 2 for x in tensor.shape ] ) )
        self.assertTrue( np.allclose( np.reshape( tensor, (8, 8) ), utry ) )

    def test_get_tensor_format_left ( self ):
        utry = unitary_group.rvs( 8 )
        gate = Gate( utry, (0, 1, 2) )
        tensor = gate.get_tensor_format( compress_left = True )
        self.assertTrue( len( tensor.shape ) == 4 )
        self.assertTrue( all( [ x == 2 for x in tensor.shape[1:] ] ) )
        self.assertTrue( tensor.shape[0] == 8 )
        self.assertTrue( np.allclose( np.reshape( tensor, (8, 8) ), utry ) )

    def test_get_tensor_format_right ( self ):
        utry = unitary_group.rvs( 8 )
        gate = Gate( utry, (0, 1, 2) )
        tensor = gate.get_tensor_format( compress_right = True )
        self.assertTrue( len( tensor.shape ) == 4 )
        self.assertTrue( all( [ x == 2 for x in tensor.shape[:-1] ] ) )
        self.assertTrue( tensor.shape[-1] == 8 )
        self.assertTrue( np.allclose( np.reshape( tensor, (8, 8) ), utry ) )

    def test_get_tensor_format_all ( self ):
        utry = unitary_group.rvs( 8 )
        gate = Gate( utry, (0, 1, 2) )
        tensor = gate.get_tensor_format( compress_left = True,
                                         compress_right = True )
        self.assertTrue( len( tensor.shape ) == 2 )
        self.assertTrue( tensor.shape[0] == 8 )
        self.assertTrue( tensor.shape[1] == 8 )
        self.assertTrue( np.allclose( tensor, utry ) )

if __name__ == "__main__":
    ut.main()

max_stars_repo_name: edyounis/QFactor

max_issues_repo_name: edyounis/qFactor

max_forks_repo_name: edyounis/qFactor

reason: import numpy,from scipy


text:
"""This module contains various experimental environments used for testing
human behavior."""
from __future__ import division
from numpy import zeros, ones
from numpy.random import choice, rand

class RevLearn(object):
    """Here we define the environment for the probabilistic reversal learning task
    """

    def __init__(self, observation_generator,
                 state_transition_generator,
                 state_duration_generator,
                 blocks=1, T=10):

        self.T = T
        self.blocks = blocks
        # set probability distribution used for generating observations
        self.ObsGen = observation_generator

        # set probability distribution used for generating state transitions
        self.StGen = state_transition_generator
        self.ns = self.StGen.shape[0]

        # set probability distribution used for generating duration transitions
        self.DurGen = state_duration_generator
        self.nd = self.DurGen.shape[0]

        # set container that keeps track the evolution of the hidden states
        self.hidden_states = zeros((blocks, T, 2), dtype=int)

    def set_hidden_states(self, hidden_states=None):
        if hidden_states is not None:
            self.hidden_states[:] = hidden_states
        else:
            for b in range(self.blocks):
                [s, d] = [choice(self.ns, p=ones(self.ns)/self.ns),
                          choice(self.nd, p=self.DurGen)]

                # evolve hidden states starting from initial state
                for t in range(self.T):
                    self.hidden_states[b, t, :] = [s, d]
                    if d > 0:
                        d = d-1
                    else:
                        [s, d] = [choice(self.ns, p=self.StGen[:, s]),
                                  choice(self.nd, p=self.DurGen)]

        self.observation = zeros(self.hidden_states.shape[:2], dtype=int)

    def generate_observations(self, t, responses):
        # return the observation for given choice
        s = self.hidden_states[:, t, 0]
        rp = self.O[s, responses]  # reward probability
        self.observation[:, t] = (rand(self.blocks) < rp).astype(int)

        return self.observation[:, t]

max_stars_repo_name: dimarkov/predchange

max_issues_repo_name: dimarkov/predchange

max_forks_repo_name: dimarkov/predchange

reason: from numpy


text:
import sys, os

import numpy as np

import ephesus
from tdpy.util import summgene
import tdpy

def make_rflxmoon( \
              # type of orbital architecture
              ## 'plan'
              ## 'bhol'
              ## 'planmoon'
              ## 'bholmoon'
              typeobar='planmoon', \
             ):
    '''
    Make relative flux light curves of stars with companions
    '''

    np.random.seed(0)

    # time axis
    time = np.linspace(0., 30., 10000)
    minmtime = np.amin(time)
    maxmtime = np.amax(time)

    # dictionary of conversion factors
    dictfact = ephesus.retr_factconv()

    # host
    ## radius of the star [R_S]
    radistar = 1.0
    ## total mass [M_S]
    massstar = 1.0

    # companions
    ## number of companions
    numbcomp = 2
    indxcomp = np.arange(numbcomp)

    ## orbital periods
    pericomp = np.array([3., 5.])
    ## mid-transit epochs
    epoccomp = np.array([200., 600.])
    ## cosine of inclination
    cosicomp = np.array([0.03, 0.05])
    ## inclination
    inclcomp = 180. / np.pi * np.arccos(cosicomp)

    pathmoon = os.environ['EPHESUS_DATA_PATH'] + '/imag/moon/'
    os.system('mkdir -p %s' % pathmoon)
    if typeobar == 'planmoon' or typeobar == 'plan':
        ## planet properties
        ## radii [R_E]
        radicomp = np.array([8., 16.])
        ## masses [M_E]
        masscomp = ephesus.retr_massfromradi(radicomp)
        ## densities [d_E]
        denscomp = masscomp / radicomp**3

    # approximate total mass of the system
    masstotl = massstar

    ## semi-major axes [AU]
    smaxcomp = ephesus.retr_smaxkepl(pericomp, masstotl)
    ## sum of radii divided by the semi-major axis
    #rsma = ephesus.retr_rsma(radicomp, radistar, smax)

    if typeobar.endswith('moon'):
        # exomoons to companions

        numbmoon = np.empty(numbcomp, dtype=int)
        radimoon = [[] for j in indxcomp]
        massmoon = [[] for j in indxcomp]
        densmoon = [[] for j in indxcomp]
        perimoon = [[] for j in indxcomp]
        epocmoon = [[] for j in indxcomp]
        smaxmoon = [[] for j in indxcomp]
        indxmoon = [[] for j in indxcomp]

        # Hill radius of the companion
        radihill = ephesus.retr_radihill(smaxcomp, masscomp / dictfact['msme'], massstar)
        # maximum semi-major axis of the moons
        maxmsmaxmoon = 0.5 * radihill

        masstotl = np.sum(masscomp) / dictfact['msme'] + massstar

        for j in indxcomp:
            # number of moons
            arry = np.arange(1, 10)
            prob = arry**(-2.)
            prob /= np.sum(prob)
            numbmoon[j] = np.random.choice(arry, p=prob)
            indxmoon[j] = np.arange(numbmoon[j])
            smaxmoon[j] = np.empty(numbmoon[j])
            # properties of the moons
            ## radii [R_E]
            radimoon[j] = radicomp[j] * tdpy.icdf_powr(np.random.rand(numbmoon[j]), 0.05, 0.3, 2.)
            ## mass [M_E]
            massmoon[j] = ephesus.retr_massfromradi(radimoon[j])
            ## densities [d_E]
            densmoon[j] = massmoon[j] / radimoon[j]**3
            # minimum semi-major axes
            minmsmaxmoon = ephesus.retr_radiroch(radicomp[j], denscomp[j], densmoon[j])

            # semi-major axes of the moons
            for jj in indxmoon[j]:
                smaxmoon[j][jj] = tdpy.icdf_powr(np.random.rand(), minmsmaxmoon[jj], maxmsmaxmoon[j], 2.)
            # orbital period of the moons
            perimoon[j] = ephesus.retr_perikepl(smaxmoon[j], masstotl)
            # mid-transit times of the moons
            epocmoon[j] = tdpy.icdf_self(np.random.rand(numbmoon[j]), minmtime, maxmtime)

        #print('smaxcomp')
        #print(smaxcomp)
        #print('radicomp')
        #print(radicomp)
        #print('pericomp')
        #print(pericomp)
        #print('radihill')
        #print(radihill)
        #print('minmsmaxmoon')
        #print(minmsmaxmoon)
        #print('maxmsmaxmoon')
        #print(maxmsmaxmoon)
        #print('masstotl')
        #print(masstotl)
        #print('smaxmoon')
        #print(smaxmoon)
        #print('perimoon')
        #print(perimoon)
        #print('')
        #raise Exception('')

    if (smaxmoon[j] > smaxcomp[j] / 1.2).any():
        print('smaxcomp[j]')
        print(smaxcomp[j])
        print('radihill[j]')
        print(radihill[j])
        print('smaxmoon[j]')
        print(smaxmoon[j])
        raise Exception('')

    strgextn = '%s' % (typeobar)

    # generate light curve
    rflx = ephesus.retr_rflxtranmodl(time, radistar, pericomp, epoccomp, inclcomp=inclcomp, massstar=massstar, \
                        radicomp=radicomp, masscomp=masscomp, \
                        perimoon=perimoon, epocmoon=epocmoon, radimoon=radimoon, \
                        #pathanim=pathmoon, strgextn=strgextn, \
                        )

    dictmodl = dict()
    dictmodl[strgextn] = dict()
    dictmodl[strgextn]['time'] = time
    dictmodl[strgextn]['lcur'] = rflx

    pathplot = ephesus.plot_lcur(pathmoon, dictmodl=dictmodl, strgextn=strgextn, \
                                    #titl=titlraww, \
                                )

globals().get(sys.argv[1])(*sys.argv[2:])

max_stars_repo_name: tdaylan/ephesus

max_issues_repo_name: tdaylan/ephesus

max_forks_repo_name: tdaylan/ephesus


text:
from hoomd import *
from hoomd import md

import math

import numpy as np

import unittest

context.initialize()

class minimize_rigid_test(unittest.TestCase):
    n = 7
    def setUp(self):
        context.initialize()

        snapshot = data.make_snapshot(N=2,particle_types=['A','B'],box=data.boxdim(L=25))
        if comm.get_rank() == 0:
            snapshot.particles.typeid[0] = 0
            snapshot.particles.typeid[1] = 1
            snapshot.particles.position[0] = (0,0,0)
            snapshot.particles.position[1] = (0,0,1)
            snapshot.particles.orientation[0]= (1,0,0,0)

            # rotate minimally away from the parallel configuration
            angle_deg = .01
            snapshot.particles.orientation[1]= (math.cos(0.5*angle_deg*math.pi/180),0,0,math.sin(0.5*angle_deg*math.pi/180))
            snapshot.particles.moment_inertia[0] = (0.5,1,1)
            snapshot.particles.moment_inertia[1] = (0.5,1,1)
        self.s = init.read_snapshot(snapshot)

        self.nl = md.nlist.cell()

        # two dumbbells
        self.s.particles.types.add('sphere')
        self.rigid = md.constrain.rigid()
        self.rigid.set_param('A', positions=[(-.5,0,0),(.5,0,0)], types=['sphere']*2)
        self.rigid.set_param('B', positions=[(-.5,0,0),(.5,0,0)], types=['sphere']*2)
        self.rigid.create_bodies()

        self.nl = md.nlist.cell()
        self.lj = md.pair.lj(nlist=self.nl,r_cut=2.5)
        self.lj.pair_coeff.set(['A','B','sphere'],['A','B'], epsilon=0, sigma=0, r_cut=False)
        self.lj.pair_coeff.set('sphere','sphere',epsilon=1,sigma=1)

    def test_minimize(self):
        # only update particle 1 (keep 0 fixed)
        typeB = group.type(name='typeB',type='B')
        fire = md.integrate.mode_minimize_fire(dt=0.005,wtol=1e-10,ftol=1e-10)
        nve = md.integrate.nve(group=typeB)
        max_conv = 10000
        log = analyze.log(quantities=['potential_energy'],period=100,filename=None)
        run(1)
        old_energy = log.query('potential_energy')
        while not(fire.has_converged()) and get_step() < max_conv:
            run(100)
        self.assertTrue(fire.has_converged())

        # check that dumbbell 1 is now forming a 'cross' with dumbbell 0
        self.assertAlmostEqual(abs(self.s.particles[1].orientation[0]),1/math.sqrt(2),6)
        self.assertTrue(abs(self.s.particles[1].orientation[1])<0.001)
        self.assertTrue(abs(self.s.particles[1].orientation[2])<0.001)
        self.assertAlmostEqual(abs(self.s.particles[1].orientation[3]),1/math.sqrt(2),6)
        self.assertTrue(log.query('potential_energy') < old_energy)

if __name__ == '__main__':
    unittest.main(argv = ['test.py', '-v'])

max_stars_repo_name: PetersResearchGroup/PCND

max_issues_repo_name: PetersResearchGroup/PCND

max_forks_repo_name: PetersResearchGroup/PCND


text:
"""
This module contains the camera base class as well as a class for the robots in hand camera and cage cam.
"""
import cv2
import pybullet as p
import math
import numpy as np

class Camera:
    """
    Camera base class.
    """

    def __init__(self):
        # TODO Load from yaml for different camera configurations
        self.intrinsics = np.array(
            [[596.6278076171875, 0.0, 311.98663330078125, 0.0],
             [0.0, 596.6278076171875, 236.76170349121094, 0.0],
             [0.0, 0.0, 1.0, 0.0]])

        self.width = 640
        self.height = 480
        self.fx = self.intrinsics[0, 0]  # focal length
        self.fy = self.intrinsics[1, 1]
        self.cx = self.intrinsics[0, 2]  # principal point
        self.cy = self.intrinsics[1, 2]

        self.fov = 2 * math.atan(self.height / 2 / self.fy) * 180 / math.pi  # field of view angle
        # self.fov = 2*math.atan(self.pixel_width/2/self.intrinsics[0, 0])*180/math.pi  # field of view angle
        self.near = 0.01
        self.far = 10
        self.init_camera_vector = (0, 0, 1)  # z axis
        self.init_up_vector = (0, -1, 0)
        self.plt = None
        self.fig = None
        self.ax = None
        self.lookat = None

        # cf https://blog.noctua-software.com/opencv-opengl-projection-matrix.html
        # except at (1, 1) where he put a wrong minus
        self.projection_matrix = (2.0 * self.fx / self.width, 0.0, 0.0, 0.0,
                                  0.0, 2.0 * self.fy / self.height, 0.0, 0.0,
                                  1.0 - 2.0 * self.cx / self.width, 2.0 * self.cy / self.height - 1.0,
                                  (self.far + self.near) / (self.near - self.far), -1.0,
                                  0.0, 0.0, 2.0 * self.far * self.near / (self.near - self.far), 0.0)
        self.eye_in_hand_camera_projection_matrix = self.projection_matrix

    def getPosRot(self, id, client_id):
        pos = None
        rot = None
        rot_matrix_matrix = None
        return pos, rot, rot_matrix_matrix

    def get_view_matrix(self, id, client_id):
        pos, rot, rot_matrix = self.getPosRot(id, client_id)

        camera_vector = rot_matrix.dot(self.init_camera_vector)
        up_vector = rot_matrix.dot(self.init_up_vector)

        camera_eye_pos = np.array(pos)
        camera_target_position = camera_eye_pos + 0.2 * camera_vector
        self.lookat = camera_target_position

        view_matrix = p.computeViewMatrix(camera_eye_pos,
                                          camera_target_position,
                                          up_vector)

        return view_matrix

    def calc_point_cloud(self, id, client_id, with_noise=False):
        """
        returns the 3d world coordinates and the corresponding rgb values of inHandCamera.
        :param client_id: id of the client simulating in
        :param id: if inHandCam -> equal to robot_id in the physics client
                   if CageCam -> equal to id of the cam in the physics client
        :param plot_points: whether the points shall be plot in matplotlib (very slow)
        :return: points: np array (nb_pointsx3)
                 colors: np array (nb_pointsx4)
        """
        rgb_img, z, _ = self.get_image(id, client_id, with_noise=with_noise)

        u = np.arange(self.width) - self.cx
        v = np.arange(self.height) - self.cy

        x = (z * u) / self.fx
        y = (z.T * v).T / self.fy

        points = np.stack((x, y, z), axis=-1).reshape((self.width * self.height, 3))
        colors = rgb_img.reshape((self.width * self.height, 4))

        # if plot_points:
        #     if self.plt is None:
        #         import matplotlib.pyplot as plt
        #         self.plt = plt
        #         self.fig = plt.figure()
        #         self.ax = self.fig.add_subplot(111, projection='3d')
        #
        #     self.ax.scatter(points[:, 0], points[:, 1], points[:, 2], c=colors)
        #     self.plt.show()
        #     self.plt.pause(0.01)

        # returns all points in points (nb_points x 3) and the correspoinding rgb colors (nb_points x 4) [r,g,b,1]
        return points, colors

    def clear_points_in_plot(self):
        """
        Clear the current axis of the plot.

        :return: no return value
        """
        self.plt.cla()

    def remove_created_bodies(self, list_ids, n_bodies):
        for i in range(n_bodies):
            p.removeBody(list_ids[i])
        p.configureDebugVisualizer(p.COV_ENABLE_RENDERING, 1)

    def get_image(self, cam_id, client_id, with_noise=False, shadow=True):
        """
        Returns the rgb, depth and segamsks as np array of the inHand camera.

        :param: robot_id: robot id which carries the inhand camera
        :param: client_id: id of the client simulating in
        :return: np array of rgb, depth, segmask images
        """
        view_matrix = self.get_view_matrix(cam_id, client_id)

        if shadow:
            _, _, rgb_img, depth_img, seg_img = p.getCameraImage(
                self.width, self.height,
                view_matrix, self.projection_matrix,
                shadow=0, lightDirection=[1, 1, 1],
                renderer=p.ER_BULLET_HARDWARE_OPENGL,
                physicsClientId=client_id)
        else:
            _, _, rgb_img, depth_img, seg_img = p.getCameraImage(
                self.width, self.height,
                view_matrix, self.projection_matrix,
                shadow=1, lightDirection=[1, 1, 1],
                renderer=p.ER_TINY_RENDERER,
                physicsClientId=client_id)

        rgb_img = np.array(rgb_img) / 255.

        # do not use this
        # rgb_img = np.reshape(rgb_img, (self.height, self.width, 4))
        # depth_img = np.reshape(depth_img, (self.height, self.width))
        # seg_img = np.reshape(seg_img, (self.height, self.width))

        # From normalized to actual depth, as described here:
        # http://web.archive.org/web/20130416194336/http://olivers.posterous.com/linear-depth-in-glsl-for-real
        z = 2 * self.far * self.near / (self.far + self.near - (self.far - self.near) * (2 * np.array(depth_img) - 1))

        if with_noise:
            # Gaussian noise (scale with power of 2 of the real depth + offset)
            depth_img = depth_img + (0.0001 * np.power(z - 0.5, 2) + 0.0004) * np.random.rand(self.height, self.width)

            # Quantization (scale with the inverse depth)
            depth_img = ((40000 * depth_img).astype(int) / 40000.).astype(np.float32)

            # From normalized to actual depth, with noise
            z = 2 * self.far * self.near / (self.far + self.near - (self.far - self.near) * (2 * depth_img - 1))

            # Final smoothing with a bilateral filter
            # args: kernel size, sigma in "color" space, sigma in coordinate space
            z = cv2.bilateralFilter(z, 5, 0.1, 5)

        return rgb_img, z, seg_img

class InHandCamera(Camera):
    """
    In hand camera of the robot. Extends camera base class.
    """

    def __init__(self):
        Camera.__init__(self)

    def getPosRot(self, cam_id, client_id):
        pos, rot, _, _, _, _ = p.getLinkState(cam_id, linkIndex=20,
                                              computeForwardKinematics=True,
                                              physicsClientId=client_id)
        rot_matrix = p.getMatrixFromQuaternion(rot)
        rot_matrix = np.array(rot_matrix).reshape(3, 3)
        return pos, rot, rot_matrix

class CageCamera(Camera):
    """
    Cage camera. Extends the camera base class.
    """

    def __init__(self):
        Camera.__init__(self)

    def getPosRot(self, id, client_id):
        pos, rot = p.getBasePositionAndOrientation(id, physicsClientId=client_id)
        rot_matrix = p.getMatrixFromQuaternion(rot)
        rot_matrix = np.array(rot_matrix).reshape(3, 3)
        return pos, rot, rot_matrix

max_stars_repo_name: Yucheng-Tang/SimulationFrameworkPublic

max_issues_repo_name: Yucheng-Tang/SimulationFrameworkPublic

max_forks_repo_name: Yucheng-Tang/SimulationFrameworkPublic


text:
import pandas as pd
import glob
from sklearn.metrics import roc_auc_score
import numpy as np
from functools import partial
from scipy.optimize import fmin
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler

import xgboost as xgb

def run_training(pred_df, fold):
    train_df = pred_df[pred_df.kfold != fold].reset_index(drop=True)
    valid_df = pred_df[pred_df.kfold == fold].reset_index(drop=True)
    xtrain = train_df[["lr_pred", "lr_cnt_pred", "rf_svd_pred"]].values
    xvalid = valid_df[["lr_pred", "lr_cnt_pred", "rf_svd_pred"]].values

    scl = StandardScaler()
    xtrain = scl.fit_transform(xtrain)
    xvalid = scl.transform(xvalid)

    opt = xgb.XGBClassifier(random_state=42, tree_method='gpu_hist')
    opt.fit(xtrain, train_df.sentiment.values)
    pred = opt.predict_proba(xvalid)[:, 1]
    auc = roc_auc_score(valid_df.sentiment.values, pred)
    valid_df.loc[:, 'xgb_pred'] = pred
    print(f'Fold : {fold}, AUC : {auc}')
    return valid_df

if __name__ == '__main__':
    files = glob.glob('../model_pred/*.csv')
    df = None
    for f in files:
       if df is None:
           df = pd.read_csv(f)
       else:
           temp_df = pd.read_csv(f)
           df = df.merge(temp_df, on="id", how="left")
    targets = df.sentiment.values
    pred_col = ["lr_pred", "lr_cnt_pred", "rf_svd_pred"]

    dfs = []
    for j in range(5):
        temp_df = run_training(df, j)
        dfs.append(temp_df)

    fin_valid_df = pd.concat(dfs)
    print(roc_auc_score(targets, fin_valid_df.xgb_pred.values))


text:
import numpy as np
import cupy as cp
from cupy.cuda import Device
from time import perf_counter

# params
num_iter = 1000
gpus = 4
N = 4
local_N = N // gpus
T = np.float64
shape = (N, N)

# matrix, vector
A_list = [None] * gpus
b_list = [None] * gpus
b = cp.ones(N, T)
y = cp.zeros(N, T)
y_list = [None] * gpus

def gen(i):
	arr = []
	for j in range(local_N):
		begin = (i*local_N+j)*N
		end = (i*local_N+j+1)*N
		arr.append(np.arange(begin, end, dtype=np.float64))
	return arr

# init device
for i in range(gpus):
	Device(i).use()
	pool = cp.cuda.MemoryPool(cp.cuda.malloc_managed)
	cp.cuda.set_allocator(pool.malloc)

# P2P
for i in range(gpus):
	Device(i).use()
	for j in range(gpus):
		if i == j:
			continue
		flag = cp.cuda.runtime.deviceCanAccessPeer(i, j)
		res = 'Enable' if flag else 'Disable'
		print(f'{i} -> {j}: {res}')
		cp.cuda.runtime.deviceEnablePeerAccess(j)

# alloc
Device(0).use()
b = cp.ones(N, dtype=T)
for i in range(gpus):
	Device(i).use()
	A_list[i] = cp.array(gen(i), T)
	# A_list[i] = cp.ones(shape, T)
	A_list[i] = cp.arange(N*i, N*(i+1), dtype=T)
	# b_list[i] = cp.arange(N*i, N*(i+1), dtype=T)
	# y_list[i] = cp.zeros(N, T)
	y_list[i] = cp.zeros(local_N, T)

# dot
for i in range(gpus):
	Device(i).use()
	y_list[i] = A_list[i].dot(b)

# result
for i in range(gpus):
	Device(i).synchronize()
	print(y_list[i])

reason: import numpy,import cupy,from cupy


text:
import cv2 as cv
import numpy as np
import sys
import math

def detect(path):
    img_edges = _find_edges(path)
    lines = _find_four_lines(img_edges)
    corners = _find_four_corners(lines)

    original = cv.imread(path)
    cropped_image = _persp_transform(original, corners)

    return cropped_image

def _find_edges(path):
    src = cv.imread(path)

    img_gray = cv.cvtColor(src, cv.COLOR_BGR2GRAY)

    #img_blured = cv.medianBlur(img_gray, 43)
    #edges = cv.Canny(img_blured, 10, 50, apertureSize=3)
    img_blured = cv.medianBlur(img_gray, 101)
    edges = cv.Canny(img_blured, 10, 50, apertureSize=3)

    kernel = cv.getStructuringElement(cv.MORPH_ELLIPSE, (3, 3))
    dilated = cv.dilate(edges, kernel)

    img_with_border = cv.copyMakeBorder(dilated, 10, 10, 10, 10,
                                        cv.BORDER_CONSTANT, value=[255, 255, 255])

    return img_with_border

def _find_four_lines(img):
    lines = cv.HoughLines(img, 1, math.pi/180.0, 250, np.array([]), 0, 0)

    if lines is not None:
        average_v_angle_dev = _find_average_vertical_angle_deviation(lines)
        lines = [_find_top(lines, img, average_v_angle_dev),
                 _find_bottom(lines, img, average_v_angle_dev),
                 _find_left(lines, img, average_v_angle_dev),
                 _find_right(lines, img, average_v_angle_dev)]
        return lines
    return None

def _find_four_corners(lines):
    return np.array(_get_corners(lines))

def _persp_transform(img, s):
    s_points = np.array([s[0], s[2], s[3], s[1]])

    height = max(np.linalg.norm(s_points[0] - s_points[1]),
                 np.linalg.norm(s_points[2] - s_points[3]))
    width = max(np.linalg.norm(s_points[1] - s_points[2]),
                np.linalg.norm(s_points[3] - s_points[0]))

    t_points = np.array([[0, 0],
                         [0, height],
                         [width, height],
                         [width, 0]], np.float32)

    if s_points.dtype != np.float32:
        s_points = s_points.astype(np.float32)

    M = cv.getPerspectiveTransform(s_points, t_points)
    return cv.warpPerspective(img, M, (int(width), int(height)))

# --------------------------------------------------------------------------

def _get_corners(polar):
    result = [_calc_point(polar, 0, 2),
              _calc_point(polar, 0, 3),
              _calc_point(polar, 1, 2),
              _calc_point(polar, 1, 3)]

    return result

def _calc_point(polar, index1, index2):
    aa = np.empty((0, 2), float)
    bb = np.empty((0, 1), float)

    rho = polar[index1][0]
    theta = polar[index1][1]
    a = math.cos(theta)
    b = math.sin(theta)

    aa = np.vstack([aa, [a, b]])
    bb = np.append(bb, rho)

    # ---------------------

    rho = polar[index2][0]
    theta = polar[index2][1]
    a = math.cos(theta)
    b = math.sin(theta)

    aa = np.vstack([aa, [a, b]])
    bb = np.append(bb, rho)

    t = np.linalg.solve(aa, bb)
    # if index1 == 0 and index2 == 2:
    #    return [int(t[0]+60), int(t[1]+60)]
    # if index1 == 0 and index2 == 3:
    #    return [int(t[0]-60), int(t[1]+60)]
    # if index1 == 1 and index2 == 2:
    #    return [int(t[0]+60), int(t[1]-60)]
    # if index1 == 1 and index2 == 3:
    #    return [int(t[0]-60), int(t[1]-60)]

    return (int(t[0]), int(t[1]))

def _find_average_vertical_angle_deviation(lines):
    a, b, c = lines.shape

    theta_interval1 = [0, 20]
    theta_interval2 = [160, 180]

    theta_sum1 = 0
    theta_sum2 = 0

    count1 = 0
    count2 = 0

    for i in range(a):
        theta = lines[i][0][1]*180/math.pi
        if (theta > theta_interval1[0] and theta < theta_interval1[1]):
            theta_sum1 += theta
            count1 += 1

        if (theta > theta_interval2[0] and theta < theta_interval2[1]):
            theta_sum2 += theta
            count2 += 1

    average_angle1 = theta_sum1/count1

    average_angle2 = 180 - theta_sum2/count2

    return (average_angle1 + average_angle2)/2

def _find_top(lines, img, average_v_angle_dev):
    a, b, c = lines.shape
    h, w = img.shape

    max_rho = 0
    found_theta = 0
    max_search = h/2

    #def_angle = 90 + average_v_angle_dev
    def_angle = 90

    theta_interval = [def_angle-1, def_angle+1]

    for i in range(a):
        rho = lines[i][0][0]
        theta = lines[i][0][1]*180/math.pi
        if (rho > max_rho and rho < max_search):

            if (theta > theta_interval[0] and theta < theta_interval[1]):
                max_rho = rho
                found_theta = theta

    return (max_rho, found_theta*math.pi/180)

def _find_bottom(lines, img, average_v_angle_dev):
    a, b, c = lines.shape
    h, w = img.shape

    min_rho = h
    found_theta = 0
    max_search = h/2

    #def_angle = 90 + average_v_angle_dev
    def_angle = 90

    theta_interval = [def_angle-1, def_angle+1]

    for i in range(a):
        rho = lines[i][0][0]
        theta = lines[i][0][1]*180/math.pi
        if (rho < min_rho and rho > max_search):

            if (theta > theta_interval[0] and theta < theta_interval[1]):
                min_rho = rho
                found_theta = theta

    return (min_rho, found_theta*math.pi/180)

def _find_left(lines, img, average_v_angle_dev):
    a, b, c = lines.shape
    h, w = img.shape

    max_rho = 0
    found_theta = 0
    max_search = w/2

    #def_angle1 = 0 + average_v_angle_dev
    #def_angle2 = 180 - average_v_angle_dev

    def_angle1 = 0
    def_angle2 = 180

    theta_interval1 = [def_angle1-2, def_angle1+2]
    theta_interval2 = [def_angle2-2, def_angle2+2]

    for i in range(a):
        rho = lines[i][0][0]
        theta = lines[i][0][1]*180/math.pi

        if (rho > max_rho and rho < max_search):

            if ((theta > theta_interval1[0] and theta < theta_interval1[1]) or
                    (theta > theta_interval2[0] and theta < theta_interval2[1])):
                max_rho = rho
                found_theta = theta

    return (max_rho, found_theta*math.pi/180)

def _find_right(lines, img, average_v_angle_dev):
    a, b, c = lines.shape
    h, w = img.shape

    min_rho = w
    found_theta = 0
    negative = False

    min_search = w/2

    #def_angle1 = 0 + average_v_angle_dev
    #def_angle2 = 180 - average_v_angle_dev

    def_angle1 = 0
    def_angle2 = 180

    theta_interval1 = [def_angle1-2, def_angle1+2]
    theta_interval2 = [def_angle2-2, def_angle2+2]

    for i in range(a):
        rho = lines[i][0][0]
        theta = lines[i][0][1]*180/math.pi

        if (abs(rho) < min_rho and abs(rho) > min_search):
            if ((theta > theta_interval1[0] and theta < theta_interval1[1]) or
                    (theta > theta_interval2[0] and theta < theta_interval2[1])):
                min_rho = abs(rho)
                negative = rho < 0

                found_theta = theta

    return ((-1 if negative else 1) * min_rho, found_theta*math.pi/180)

max_stars_repo_name: lazarmarkovic/cursive-cyrillic-htr

max_issues_repo_name: lazarmarkovic/cursive-cyrillic-htr

max_forks_repo_name: lazarmarkovic/cursive-cyrillic-htr