text:
from __future__ import print_function
from builtins import zip
from builtins import str
from builtins import range
from builtins import object
import os
import re
import ast
import glob
import hashlib
from pathlib import Path
from typing import Union

import pandas
import numpy as np
import datetime
import subprocess
from ast import literal_eval as make_tuple

import sibispy
from sibispy import sibislogger as slog
from sibispy import utils as sutils
from sibispy import config_file_parser as cfg_parser
from sibispy.cluster_util import SlurmScheduler

class redcap_to_casesdir(object):
    def __init__(self):
        self.__import_forms = dict()
        self.__export_forms = dict()
        self.__export_rename = dict()
        # Make lookup dicts for mapping radio/dropdown codes to labels
        self.__code_to_label_dict = dict()
        self.__metadata_dict = dict()
        self.__event_dict = dict()
        self.__demographic_event_skips = list()
        self.__forms_dir =  None
        self.__sibis_defs = None
        self.__scanner_dict = None

    def configure(self, sessionObj, redcap_metadata):
        # Make sure it was set up correctly
        if not sessionObj.get_ordered_config_load() :
            slog.info('recap_to_cases_dir.configure',"ERROR: session has to be configured with ordered_config_load set to True")
            return False

        # reading script specific settings
        (cfgParser,err_msg) = sessionObj.get_config_sys_parser()
        if err_msg:
            slog.info('recap_to_cases_dir.configure',str(err_msg))
            return False

        self.__sibis_defs = cfgParser.get_category('redcap_to_casesdir')

        self.__scanner_dict = self.__sibis_defs['scanner_dict']
        for TYPE in list(self.__scanner_dict.keys()) :
            self.__scanner_dict[TYPE] = self.__scanner_dict[TYPE].split(",")

        # Reading in events
        self.__event_dict = self.__transform_dict_string_into_tuple__('event_dictionary')
        if not  self.__event_dict:
            return False

        # Reading in which events to skip demographics generation for (i.e. midyears)
        self.__demographic_event_skips = self.__sibis_defs['skip_demographics_for']

        # reading in all forms and variables that should be exported to cases_dir
        self.__forms_dir  = os.path.join(sessionObj.get_operations_dir(),'redcap_to_casesdir')
        if not os.path.exists(self.__forms_dir) :
            slog.info('redcap_to_casesdir.configure','ERROR: ' + str(self.__forms_dir)  + " does not exist!")
            return False

        exports_files = glob.glob(os.path.join(self.__forms_dir, '*.txt'))

        for f in exports_files:
            file = open(f, 'r')
            contents = [line.strip() for line in file.readlines()]
            file.close()

            export_name = re.sub('\.txt$', '', os.path.basename(f))
            import_form = re.sub('\n', '', contents[0])
            self.__import_forms[export_name] = import_form
            self.__export_forms[export_name] = [re.sub('\[.*\]', '', field) for field in contents[1:]] + ['%s_complete' % import_form]
            self.__export_rename[export_name] = dict()

            for field in contents[1:]:
                match = re.match('^(.+)\[(.+)\]$', field)
                if match:
                    self.__export_rename[export_name][match.group(1)] = match.group(2)

        return self.__organize_metadata__(redcap_metadata)

    def __transform_dict_string_into_tuple__(self,dict_name):
        dict_str = self.__sibis_defs[dict_name]
        dict_keys = list(dict_str.keys())
        if not len(dict_keys):
            slog.info('redcap_to_casesdir.configure',"ERROR: Cannot find '" + dict_name + "'' in config file!")
            return None

        dict_tup = dict()
        for key in dict_keys:
            # turn string into tuple
            dict_tup[key] = make_tuple("(" + dict_str[key] +")")

        return dict_tup

    # Organize REDCap metadata (data dictionary)
    def __organize_metadata__(self,redcap_metadata):
        # turn metadata into easily digested dict
        for field in redcap_metadata:
            field_tuple = (field['field_type'],
                           field['text_validation_type_or_show_slider_number'],
                           field['field_label'],
                           field['text_validation_min'],
                           field['text_validation_max'],
                           field['select_choices_or_calculations'])

            self.__metadata_dict[field['field_name']] = field_tuple

        meta_data_dict = self.__transform_dict_string_into_tuple__('general_datadict')
        if not  meta_data_dict :
            return False

        self.__metadata_dict.update(meta_data_dict)

        if not self.__check_all_forms__():
            return False

        self.__make_code_label_dict__(redcap_metadata)
        return True

    # Filter confidential fields from all forms
    def __check_all_forms__(self):
        # Filter each form
        text_list = list()
        non_redcap_list = list()
        for export_name in list(self.__export_forms.keys()):
            (form_text_list, form_non_redcap_list)  = self.__check_form__(export_name)
            if form_text_list :
                text_list += form_text_list
            if form_non_redcap_list:
                non_redcap_list += form_non_redcap_list

        if text_list:
            slog.info('redcap_to_casesdir.__check_all_forms__.' + hashlib.sha1(str(text_list).encode()).hexdigest()[0:6], "ERROR: The txt file(s) in '" + str(self.__forms_dir) + "' list non-numeric redcap variable names!",
                      form_variable_list = str(text_list),
                      info = "Remove it from form file or modify definition in REDCap")

        if non_redcap_list :
            slog.info('redcap_to_casesdir.__check_all_forms__.' +  hashlib.sha1(str(text_list).encode()).hexdigest()[0:6], "ERROR: The txt file(s) in '" + str(self.__forms_dir) + "' list variables that do not exist in redcap!",
                      form_variable_list = str(non_redcap_list),
                      info = "Remove it from form or modify definition REDCap")

        if non_redcap_list or text_list:
            return False

        return True

    # Filter potentially confidential fields out of given list, based on project
    #  metadata
    def __check_form__(self, export_name):
        text_list = list()
        non_redcap_list = list()

        for field_name in self.__export_forms[export_name]:
            try:
                (field_type, field_validation, field_label, text_val_min,
                 text_val_max, choices) = self.__metadata_dict[re.sub('___.*', '', field_name)]
                if (field_type != 'text' and field_type != 'notes') or (field_validation in ['number', 'integer', 'time']):
                    pass
                else:
                    text_list.append([export_name,field_name, field_type, field_validation])
            except:
                if '_complete' not in field_name:
                    non_redcap_list.append([export_name,field_name])

        return (text_list,non_redcap_list)

    def __make_code_label_dict__(self,redcap_metadata):
        # First turn metadata into easily digested dict
        for field in redcap_metadata:
            if field['field_type'] in ['radio', 'dropdown']:
                field_dict = {'': ''}
                choices = field['select_choices_or_calculations']
                for choice in choices.split('|'):
                    code_label = [c.strip() for c in choice.split(',')]
                    field_dict[code_label[0]] = ', '.join(code_label[1:])
                self.__code_to_label_dict[field['field_name']] = field_dict

    # used to be get_export_form_names
    def get_export_names_of_forms(self):
        return list(self.__export_forms.keys())

    def create_datadict(self, export_name, datadict_dir):
         if export_name not in self.__export_forms.keys() :
             slog.info('redcap_to_casesdir.create_datadict',"ERROR: could not create data dictionary for form " + export_name)
             return None

         export_form_entry_list = self.__export_forms[export_name]
         size_entry_list = len(export_form_entry_list)
         export_form_list = [export_name] * size_entry_list
         return self.__create_datadicts_general__(datadict_dir, export_name, export_form_list,export_form_entry_list)

    # defining entry_list only makes sense if export_forms_list only consists of one
    # entry !
    def create_all_datadicts(self, datadict_dir):
        for export_name in self.get_export_names_of_forms():
            self.create_datadict(export_name,datadict_dir)
        self.create_demographic_datadict(datadict_dir)

    # Create custom form for demographics
    def create_demographic_datadict(self, datadict_dir):
        meta_data_dict = self.__transform_dict_string_into_tuple__('demographic_datadict')
        if not meta_data_dict:
            return False
        self.__metadata_dict.update(meta_data_dict)

        dict_str = self.__sibis_defs['demographic_datadict']
        export_entry_list = list(dict_str.keys())

        export_form_list = ['demographics'] * len(export_entry_list)

        return self.__create_datadicts_general__(datadict_dir, 'demographics', export_form_list,export_entry_list)

    # for each entry in the form list you have to define a variable
    def __create_datadicts_general__(self,datadict_dir, datadict_base_file,export_forms_list, variable_list):
        redcap_datadict_columns = ["Variable / Field Name", "Form Name",
                                   "Section Header", "Field Type", "Field Label",
                                   "Choices, Calculations, OR Slider Labels",
                                   "Field Note",
                                   "Text Validation Type OR Show Slider Number",
                                   "Text Validation Min", "Text Validation Max",
                                   "Identifier?",
                                   "Branching Logic (Show field only if...)",
                                   "Required Field?", "Custom Alignment",
                                   "Question Number (surveys only)",
                                   "Matrix Group Name", "Matrix Ranking?"]

        # Insert standard set of data elements into each datadict.
        for i in range(3):
            elements = ['subject', 'arm', 'visit']
            export_forms_list.insert(i, export_forms_list[0])
            variable_list.insert(i, elements[i])

        if not os.path.exists(datadict_dir):
            os.makedirs(datadict_dir)

        ddict = pandas.DataFrame(index=variable_list,columns=redcap_datadict_columns)

        for name_of_form, var in zip(export_forms_list, variable_list):
            field_name = re.sub('___.*', '', var)
            ddict["Variable / Field Name"][var] = field_name
            ddict["Form Name"][var] = name_of_form

            # Check if var is in data dict ('FORM_complete' fields are NOT)
            if field_name in list(self.__metadata_dict.keys()):
                ddict["Field Type"][var] = self.__metadata_dict[field_name][0]
                # need to transfer to utf-8 code otherwise can create problems when
                # writing dictionary to file it just is a text field so it should not matter
                #  .encode('utf-8')
                # Not needed in Python 3 anymore
                ddict["Field Label"][var] = self.__metadata_dict[field_name][2]
                ddict["Text Validation Type OR Show Slider Number"][var] = self.__metadata_dict[field_name][1]
                ddict["Text Validation Min"][var] = self.__metadata_dict[field_name][3]
                ddict["Text Validation Max"][var] = self.__metadata_dict[field_name][4]
                #.encode('utf-8')
                ddict["Choices, Calculations, OR Slider Labels"][var] = self.__metadata_dict[field_name][5]

        # Finally, write the data dictionary to a CSV file
        dicFileName = os.path.join(datadict_dir,datadict_base_file + '_datadict.csv')
        try:
            sutils.safe_dataframe_to_csv(ddict,dicFileName)
            return dicFileName
        except Exception as err_msg:
            slog.info('redcap_to_casesdir.__create_datadicts_general__',"ERROR: could not export dictionary" + dicFileName,
                      err_msg = str(err_msg))
            return None

    # Truncate age to 2 digits for increased identity protection
    def __truncate_age__(self, age_in):
        matched = re.match('([0-9]*\.[0-9]*)', str(age_in))
        if matched:
            return round(float(matched.group(1)), 2)
        else:
            return age_in

    def __get_scanner_mfg_and_model__(self, mri_scanner, expid):
        if mri_scanner == 'nan' :
            return ["",""]

        mri_scanner= mri_scanner.upper()
        for TYPE in list(self.__scanner_dict.keys()) :
            if TYPE in mri_scanner :
                return self.__scanner_dict[TYPE]

        slog.info(expid, "Error: Do not know scanner type", script='redcap_to_casesdir.py', mri_scanner = mri_scanner)
        return ["",""]

    # NCANDA SPECIFIC - Generalize later
    # Create "demographics" file "by hand" - this includes some text fields
    def export_subject_demographics(
        self,
        subject,
        subject_code,
        arm_code,
        visit_code,
        site,
        visit_age,
        subject_data,
        visit_data,
        exceeds_criteria_baseline,
        siblings_enrolled_yn_corrected,
        siblings_id_first_corrected,
        measures_dir,
        conditional=False,
        verbose=False,
    ):
            # Latino and race coding arrives here as floating point numbers; make
            # int strings from that (cannot use "int()" because it would fail for
            # missing data
            hispanic_code = re.sub('(.0)|(nan)', '', str(subject_data['hispanic']))
            race_code = re.sub('(.0)|(nan)', '', str(subject_data['race']))

            # scanner manufacturer map
            scanner_mfg, scanner_model = self.__get_scanner_mfg_and_model__(str(visit_data['mri_scanner']), subject + "-" + visit_code)

            # Definig enroll_exception_drinking_2
            if exceeds_criteria_baseline < 0 :
                exceeds_criteria_baseline=int(subject_data['enroll_exception___drinking'])

            if siblings_enrolled_yn_corrected < 0:
                siblings_enrolled_yn_corrected=subject_data['siblings_enrolled___true']

            # No sibling than by default it is the subject itself
            if siblings_enrolled_yn_corrected == 0 :
                siblings_id_first_corrected = subject_code
            elif siblings_id_first_corrected == None :
                # if there is a sibling and if not a special case, then use default
                siblings_id_first_corrected=subject_data['siblings_id1']
                # unless not defined either -> then it must be the first subject
                if type(siblings_id_first_corrected) is not str or siblings_id_first_corrected == "" :
                    siblings_id_first_corrected = subject_code

            # if you add a line pe
            if race_code == '6':
                # if other race is specified, mark race label with manually curated
                # race code
                race_label=subject_data['race_other_code']
            else :
                race_label=self.__code_to_label_dict['race'][race_code]

            if pandas.isnull(subject_data['family_id']):
                family_id = ""
            else:
                family_id = str(int(subject_data['family_id']))

            if conditional:
                visit_age = ''
            else:
                visit_age = self.__truncate_age__(visit_age)

            demographics = [
                ['subject', subject_code],
                ['arm', arm_code],
                ['visit', visit_code],
                ['site', site],
                ['sex', subject[8]],
                ['visit_age', visit_age],
                ['mri_structural_age', self.__truncate_age__(visit_data['mri_t1_age'])],
                ['mri_diffusion_age', self.__truncate_age__(visit_data['mri_dti_age'])],
                ['mri_restingstate_age', self.__truncate_age__(visit_data['mri_rsfmri_age'])],
                ['exceeds_bl_drinking',
                 'NY'[int(subject_data['enroll_exception___drinking'])]],
                ['exceeds_bl_drinking_2',exceeds_criteria_baseline],
                ['siblings_enrolled_yn',
                 'NY'[int(subject_data['siblings_enrolled___true'])]],
                ['siblings_id_first', subject_data['siblings_id1']],
                ['siblings_enrolled_yn_2',
                 'NY'[int(siblings_enrolled_yn_corrected)]],
                # ['siblings_id_first_2', siblings_id_first_corrected],
                ['family_id', family_id],
                ['hispanic', self.__code_to_label_dict['hispanic'][hispanic_code][0:1]],
                ['race', race_code],
                ['race_label', race_label],
                ['participant_id', subject],
                ['scanner', scanner_mfg],
                ['scanner_model', scanner_model],
            ]

            series = pandas.Series()
            for (key, value) in demographics:
                series.at[key] = value

            target_path = os.path.join(measures_dir, 'demographics.csv')
            if conditional and os.path.exists(target_path):
                pass
            else:
                return sutils.safe_dataframe_to_csv(pandas.DataFrame(series).T,
                                                    target_path,
                                                    verbose=verbose)

    def export_subject_form(self, export_name, subject, subject_code, arm_code, visit_code, all_records, measures_dir,verbose = False):
        # Remove the complete field from the list of forms
        complete = '{}_complete'.format(self.__import_forms.get(export_name))
        fields = [column for column in self.__export_forms.get(export_name)
                  if column != complete]

        # Select data for this form - "reindex" is necessary to put
        # fields in listed order - REDCap returns them lexicographically sorted
        fields = [i for i in fields if i not in ['subject', 'arm', 'visit']]
        record = all_records[fields].reindex(fields, axis=1)

        # if I read it correctly then this statement is not possible
        if len(record) > 1:
            slog.info(subject + "-" + visit_code, "ERROR: muliple records for that visit found for form '" + export_name + "'!" )
            return None

        # Nothing to do
        if not len(record):
            if verbose :
                slog.info(subject  + "-" + visit_code, "Info: visit data did not contain records of form '" + export_name + "'!" )

            return None

        # First, add the three index columns
        record.insert(0, 'subject', subject_code)
        record.insert(1, 'arm', arm_code)
        record.insert(2, 'visit', visit_code)

        field_idx = 0
        output_fields = []
        for field in record.columns:
            # Rename field for output if necessary
            if field in list(self.__export_rename[export_name].keys()):
                output_field = self.__export_rename[export_name][field]
            else:
                output_field = field
            output_fields.append(output_field)

            # If this is an "age" field, truncate to 2 digits for privacy
            if re.match('.*_age$', field):
                record[field] = record[field].apply(self.__truncate_age__)

            # If this is a radio or dropdown field
            # (except "FORM_[missing_]why"), add a separate column for the
            # coded label
            if field in list(self.__code_to_label_dict.keys()) and not re.match('.*_why$', field):
                code = str(record[field].iloc[0])
                label = ''
                if code in list(self.__code_to_label_dict[field].keys()):
                    label = self.__code_to_label_dict[field][code]
                field_idx += 1
                record.insert(field_idx, output_field + '_label', label)
                output_fields.append(output_field + '_label')

            field_idx += 1

        # Apply renaming to columns
        record.columns = output_fields

        # Figure out path for CSV file and export this record
        return sutils.safe_dataframe_to_csv(record,os.path.join(measures_dir, export_name + '.csv'),verbose=verbose)

    # First get data for all fields across all forms in this event - this
    # speeds up transfers over getting each form separately
    def get_subject_specific_form_data(self,subject,event,forms_this_event, redcap_project,select_exports=None):
        # define fields and forms to export
        all_fields = ['study_id']
        forbidden_export_fields = ['subject', 'visit', 'arm']
        export_list = []
        for export_name in list(self.__export_forms.keys()):
            if export_name in forbidden_export_fields:
                continue
            if (self.__import_forms[export_name] in forms_this_event):
                if (not select_exports or export_name in select_exports):
                    all_fields += [re.sub('___.*', '', field_name) for field_name in self.__export_forms[export_name]]
                    export_list.append(export_name)

        # Remove the fields we are forbidden to export from REDCap
        all_fields = np.setdiff1d(all_fields, forbidden_export_fields).tolist()

        # Get data
        all_records = redcap_project.export_records(fields=all_fields,records=[subject], events=[event],format='df')

        # return results
        return (all_records,export_list)

    # Export selected REDCap data to cases dir
    def export_subject_all_forms(self,redcap_project, site, subject, event, subject_data, visit_age, visit_data, arm_code, visit_code, subject_code, subject_datadir,forms_this_event, exceeds_criteria_baseline, siblings_enrolled_yn_corrected,siblings_id_first_corrected, select_exports=None, verbose=False):
        # Do not really use this feature later
        # Mark subjects/visits that have QA completed by creating a hidden marker file
        # qafile_path = os.path.join(subject_datadir, '.qacomplete')
        # if visit_data['mri_qa_completed'] == '1':
        #    try:
        #        if not os.path.exists(qafile_path):
        #            qafile = open(qafile_path, 'w')
        #            qafile.close()
        #    except IOError as error:
        #        slog.info(subject_code,"ERROR: unable to open QA marker file in {}. {}".format(subject_datadir, error))
        # else:
        #    try:
        #        if os.path.exists(qafile_path):
        #            os.remove(qafile_path)
        #    except OSError as error:
        #        print("ERROR: unable to remove QA marker file {}. {}".format(qafile_path, error))

        # Check if the "measures" subdirectory already exists - this is where all
        # the csv files go. Create it if necessary.

        measures_dir = os.path.join(subject_datadir, 'measures')
        if not os.path.exists(measures_dir):
            os.makedirs(measures_dir)

        # Export demographics (if selected)
        conditional = event in self.__demographic_event_skips
        if not select_exports or 'demographics' in select_exports:
            self.export_subject_demographics(
                subject=subject,
                subject_code=subject_code,
                arm_code=arm_code,
                visit_code=visit_code,
                site=site,
                visit_age=visit_age,
                subject_data=subject_data,
                visit_data=visit_data,
                exceeds_criteria_baseline=exceeds_criteria_baseline,
                siblings_enrolled_yn_corrected=siblings_enrolled_yn_corrected,
                siblings_id_first_corrected=siblings_id_first_corrected,
                measures_dir=measures_dir,
                conditional=conditional,
                verbose=verbose)

        (all_records,export_list) = self.get_subject_specific_form_data(subject,event,forms_this_event, redcap_project, select_exports)
        # Now go form by form and export data
        for export_name in export_list:
            self.export_subject_form(export_name, subject, subject_code, arm_code, visit_code, all_records, measures_dir, verbose)

    # What Arm and Visit of the study is this event?
    def translate_subject_and_event( self, subject_code, event_label):
        if event_label in list(self.__event_dict.keys()):
            (arm_code,visit_code) = self.__event_dict[event_label]
        else:
            slog.info(str(subject_code),"ERROR: Cannot determine study Arm and Visit from event %s" % event_label )
            return (None,None,None)

        pipeline_workdir_rel = os.path.join( subject_code, arm_code, visit_code )
        return (arm_code,visit_code,pipeline_workdir_rel)

    def days_between_dates( self, date_from_str, date_to_str, date_format_ymd=sutils.date_format_ymd):
        return (datetime.datetime.strptime( date_to_str, date_format_ymd ) - datetime.datetime.strptime( date_from_str, date_format_ymd ) ).days

    def get_event_dictionary(self):
        return self.__event_dict

    def schedule_cluster_job(self, job_script: str, job_title: str, submit_log: Union[str, Path] = None,
                             job_log: str = '/dev/null', verbose: bool = False) -> bool:

        slurm_config = self.__sibis_defs['cluster_config']
        slurm = SlurmScheduler(slurm_config)
        return slurm.schedule_job(job_script, job_title, submit_log, job_log, verbose)

    def schedule_old_cluster_job(self,job_script, job_title,submit_log=None, job_log=None, verbose=False):
        qsub_cmd= '/opt/sge/bin/lx-amd64/qsub'
        if not os.path.exists(qsub_cmd):
            slog.info(job_title + "-" +hashlib.sha1(str(job_script).encode('utf-8')).hexdigest()[0:6],"ERROR: Failed to schedule job as '" + qsub_cmd + "' cannot be found!", job_script = str(job_script))
            return False

        sge_env = os.environ.copy()
        sge_env['SGE_ROOT'] = '/opt/sge'
        sge_param = self.__sibis_defs['old_cluster_parameters'].split(',')
        if job_log :
            sge_param += ['-o', job_log]
        else :
            sge_param += ['-o','/dev/null']

        qsub_args= [ qsub_cmd ] + sge_param + ['-N', '%s' % (job_title) ]
        #stderr=subprocess.STDOUT
        qsub_process = subprocess.Popen( qsub_args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr= subprocess.PIPE, env=sge_env)
        (stdoutdata, stderrdata) = qsub_process.communicate(str(job_script).encode('utf-8'))

        cmd_str='echo "%s" | %s\n' % (job_script," ".join(qsub_args))
        if stderrdata :
            slog.info(job_title + "-" + hashlib.sha1(str(stderrdata).encode('utf-8')).hexdigest()[0:6],"ERROR: Failed to schedule job !", cmd = cmd_str, err_msg = str(stderrdata))
            return False

        if verbose:
            print(cmd_str)
            if stdoutdata:
                print(stdoutdata.decode('utf-8'))

        if submit_log:
            with open(submit_log, "a") as myfile:
               myfile.write(cmd_str)
               myfile.write(stdoutdata.decode('utf-8'))

        return True

max_stars_repo_name: sibis-platform/sibispy

max_issues_repo_name: sibis-platform/sibispy

max_forks_repo_name: sibis-platform/sibispy


text:
from typing import List
import numpy as np
from pyrep.objects.force_sensor import ForceSensor
from pyrep.objects.shape import Shape
from pyrep.objects.joint import Joint
from pyrep.objects.proximity_sensor import ProximitySensor
from rlbench.backend.task import Task
from rlbench.backend.conditions import DetectedCondition, NothingGrasped
from rlbench.backend.conditions import JointCondition

class OpenWineBottle(Task):

    def init_task(self) -> None:
        cap_detector = ProximitySensor("cap_detector")
        self.joint = Joint('joint')
        self.force_sensor = ForceSensor('Force_sensor')
        self.cap = Shape('cap')
        self.register_success_conditions(
            [DetectedCondition(self.cap, cap_detector, negated=True)])
        self.cap_turned_condition = JointCondition(
            self.joint, np.deg2rad(150))

    def init_episode(self, index: int) -> List[str]:
        self.cap.set_parent(self.force_sensor)
        self.cap_turned = False
        return ['open wine bottle',
                'screw open the wine bottle',
                'unscrew the bottle cap then remove it from the wine bottle']

    def variation_count(self) -> int:
        return 1

    def step(self) -> None:
        if not self.cap_turned:
            self.cap_turned = self.cap_turned_condition.condition_met()[0]
            if self.cap_turned:
                self.cap.set_parent(self.joint)

    def cleanup(self):
        self.cap.set_parent(self.force_sensor)

ext: py

max_stars_repo_name: vonHartz/RLBench

max_issues_repo_name: vonHartz/RLBench

max_forks_repo_name: vonHartz/RLBench

reason: import numpy


text:
"""
Elevation Profile Computation (elev_profile.py)
===============================================

Description
"""

import numpy as np
from osgeo import gdal
from pyproj import CRS
import geopandas as gpd
import scipy.interpolate as si
from shapely.geometry import Point

from rabpro import merit_utils as mu
from rabpro import utils as ru

def main(cl_gdf, verbose=False, nrows=50, ncols=50):
    """Description

    Parameters
    ----------
    cl_gdf : GeoDataFrame
        Desc. Should have a column called 'DA' that stores drainage areas.
        Should be in EPSG 4326 for use of the Haversine formula
    verbose : bool
        Defaults to False.
    nrows : int
        Desc. Defaults to 50.
    ncols : int
        Desc. Defaults to 50.

    Returns
    -------
    cl_gdf : GeoDataFrame
        Desc.
    merit_gdf : GeoDataFrame
        Desc.

    """

    # cl_gdf should have a column called 'DA' that stores drainage areas
    # cl_gdf should be in 4326 for use of the Haversine formula...could add a
    # check and use other methods, but simpler this way.

    # Get data locked and loaded
    dps = ru.get_datapaths(rebuild_vrts=False)
    hdem_obj = gdal.Open(dps["DEM_elev_hp"])
    da_obj = gdal.Open(dps["DEM_uda"])
    fdr_obj = gdal.Open(dps["DEM_fdr"])
    w_obj = gdal.Open(dps["DEM_width"])

    if verbose:
        print("Extracting flowpath from DEM...")

    if cl_gdf.shape[0] == 1:
        intype = "point"
    else:
        intype = "centerline"  # deprecated
        raise DeprecationWarning(
            "elev_profile only supports single 'point' coordinate pairs, not multipoint 'centerlines"
        )

    # Here, we get the MERIT flowline corresponding to the centerline. If we
    # are provided only a single point, the flowline is delineated to its
    # terminal point. Otherwise, the flowline is delineated to the limits of
    # the provided centerline. Elevation and width profiles are then extracted.
    # In addition, if a centerline is provided rather than a point, it is
    # intersected against the MERIT flowline and values are interpolated along
    # its path.
    if intype == "point":
        # Trace the centerline all the way up to the headwaters
        ds_lonlat = np.array(
            [
                cl_gdf.geometry.values[0].coords.xy[0][0],
                cl_gdf.geometry.values[0].coords.xy[1][0],
            ]
        )
        if "DA" in cl_gdf.keys():
            ds_da = cl_gdf.DA.values[0]
        else:
            ds_da = None
        cr_ds_mapped, _ = mu.map_cl_pt_to_flowline(
            ds_lonlat, da_obj, nrows, ncols, ds_da
        )

        # Mapping may be impossible
        if np.nan in cr_ds_mapped:
            if verbose is True:
                print(
                    "Cannot map provided point to a flowline; no way to extract centerline."
                )
            return cl_gdf, None

        flowpath = mu.trace_flowpath(fdr_obj, da_obj, cr_ds_mapped)
        es = get_rc_values(hdem_obj, flowpath, nodata=-9999)
        wids = get_rc_values(w_obj, flowpath, nodata=-9999)

    # elif intype == "centerline":
    #     ds_lonlat = np.array(
    #         [
    #             cl_gdf.geometry.values[-1].coords.xy[0][0],
    #             cl_gdf.geometry.values[-1].coords.xy[1][0],
    #         ]
    #     )
    #     us_lonlat = np.array(
    #         [cl_gdf.geometry.values[0].coords.xy[0][0], cl_gdf.geometry.values[0].coords.xy[1][0],]
    #     )
    #     ds_da = cl_gdf.DA.values[-1]
    #     us_da = cl_gdf.DA.values[0]
    #     cr_ds_mapped, _ = mu.map_cl_pt_to_flowline(ds_lonlat, da_obj, nrows, ncols, ds_da)
    #     cr_us_mapped, _ = mu.map_cl_pt_to_flowline(us_lonlat, da_obj, nrows, ncols, us_da)
    #     flowpath = mu.trace_flowpath(fdr_obj, da_obj, cr_ds_mapped, cr_us_mapped)
    #     es = get_rc_values(hdem_obj, flowpath, nodata=-9999)
    #     wids = get_rc_values(w_obj, flowpath, nodata=-9999)

    #     # Intersect the centerline with the MERIT flowpath to interpolate values
    #     # to each centerline point

    #     # Convert the centerline vertices to a set of multilinestrings for intersection
    #     cli_gdf = gpd.GeoDataFrame(
    #         geometry=pts_to_line_segments(cl_gdf.geometry.values), crs=cl_gdf.crs
    #     )

    #     # Prepare the DEM-flowpath for intersection with the user-provided centerline
    #     # DEM is already in EPSG:4326
    #     dem_ll = ru.xy_to_coords(flowpath[1], flowpath[0], hdem_obj.GetGeoTransform())
    #     dem_cl_lonlat = [Point(d0, d1) for d0, d1 in zip(dem_ll[0], dem_ll[1])]
    #     demi_gdf = gpd.GeoDataFrame(
    #         geometry=pts_to_line_segments(dem_cl_lonlat), crs=CRS.from_epsg(4326)
    #     )

    #     # Intersect centerline with DEM-flowpath
    #     res_intersection = gpd.sjoin(cli_gdf, demi_gdf, op="intersects", how="inner")

    #     if verbose:
    #         print(
    #             "Found {} intersections between provided centerline and DEM flowpath.".format(
    #                 len(res_intersection)
    #             )
    #         )

    #     # Map each point of the centerline to the DEM-flowpath
    #     mapper = {}
    #     att_keys = ru.parse_keys(cl_gdf)
    #     if att_keys["distance"] is None:
    #         dists = compute_dists(cl_gdf)
    #     else:
    #         dists = cl_gdf[att_keys["distance"]].values
    #     for r in res_intersection.iterrows():
    #         clidx = r[0]
    #         demidx = r[1]["index_right"]

    #         # Get intersection point
    #         ls_cl = cli_gdf.geometry.values[clidx]
    #         ls_dem = demi_gdf.geometry.values[demidx]
    #         int_pt = ls_cl.intersection(ls_dem).coords.xy

    #         # Determine which point to map the intersection to by finding the closest
    #         # along centerline and DEM-flowpath
    #         us_dist_cl = ru.haversine(
    #             (ls_cl.coords.xy[1][0], int_pt[1][0]), (ls_cl.coords.xy[0][0], int_pt[0][0]),
    #         )[0]
    #         ds_dist_cl = ru.haversine(
    #             (ls_cl.coords.xy[1][1], int_pt[1][0]), (ls_cl.coords.xy[0][1], int_pt[0][0]),
    #         )[0]
    #         if us_dist_cl < ds_dist_cl:
    #             cl_idx = clidx
    #         else:
    #             cl_idx = clidx + 1

    #         us_dist_dem = ru.haversine(
    #             (ls_dem.coords.xy[1][0], int_pt[1][0]), (ls_dem.coords.xy[0][0], int_pt[0][0]),
    #         )[0]
    #         ds_dist_dem = ru.haversine(
    #             (ls_dem.coords.xy[1][1], int_pt[1][0]), (ls_dem.coords.xy[0][1], int_pt[0][0]),
    #         )[0]
    #         if us_dist_dem < ds_dist_dem:
    #             dem_idx = demidx
    #         else:
    #             dem_idx = demidx + 1
    #         mapper[cl_idx] = dem_idx

    #     # Make the elevation profile using the mapping
    #     cl_elevs = np.ones(len(cl_gdf)) * np.nan
    #     cl_wids = np.ones(len(cl_gdf)) * np.nan
    #     for k in mapper.keys():
    #         cl_elevs[k] = es[mapper[k]]
    #         cl_wids[k] = wids[mapper[k]]

    #     # Assign the first and last centerline elevation values to match the DEM
    #     # These points have already been mapped
    #     cl_elevs[0] = es[0]
    #     cl_elevs[-1] = es[-1]

    #     nans = np.isnan(cl_elevs)

    #     # Fill in the nans using a linear interpolation between known values
    #     # First find all the groups of nans that need interpolating across
    #     e_nangroups = find_nangroups(cl_elevs)
    #     w_nangroups = find_nangroups(cl_wids)

    #     # Do the interpolations
    #     cl_elevs = interpolate_nangroups(cl_elevs, dists, e_nangroups)
    #     cl_wids = interpolate_nangroups(cl_wids, dists, w_nangroups)

    #     # Store the elevation profile and distances in the centerline geodataframe
    #     cl_gdf["MERIT Elev (m)"] = cl_elevs
    #     cl_gdf["Distance (m)"] = compute_dists(cl_gdf)
    #     cl_gdf["MERIT Width (m)"] = cl_wids
    #     cl_gdf["intersected_DEM_flowline?"] = ~nans

    # Store the elevation profile and distances of the MERIT-derived flowpath
    coords_fp = ru.xy_to_coords(flowpath[1], flowpath[0], da_obj.GetGeoTransform())
    merit_gdf = gpd.GeoDataFrame(
        data={
            "geometry": [Point(x, y) for x, y in zip(coords_fp[0], coords_fp[1])][::-1],
            "Elevation (m)": es[::-1],
            "Width (m)": wids[::-1],
            "row": flowpath[0],
            "col": flowpath[1],
        },
        crs=CRS.from_epsg(4326),
    )
    merit_gdf["Distance (m)"] = compute_dists(merit_gdf)

    return cl_gdf, merit_gdf

def compute_dists(gdf):
    """Computes cumulative distance in meters between points in gdf.

    Parameters
    ----------
    gdf : GeoDataFrame
        Desc.

    Returns
    -------
    numpy.ndarray
        Desc.

    Raises
    ------
    TypeError
        If gdf does not contain shapely.geometry.Point values.
    """
    gdfc = gdf.copy()

    if type(gdfc.geometry.values[0]) is not Point:
        raise TypeError("Cannot compute distances for non-point type geometries.")

    if gdfc.crs.to_epsg() != 4326:
        gdfc = gdfc.to_crs("EPSG:4326")

    # Compute distances along the centerline for each point
    lats = [pt.coords.xy[1][0] for pt in gdf.geometry.values]
    lons = [pt.coords.xy[0][0] for pt in gdf.geometry.values]
    ds = ru.haversine(lats, lons)
    ds = np.insert(ds, 0, 0)
    dists = np.cumsum(ds)

    return dists

def get_rc_values(gdobj, rc, nodata=-9999):
    """Returns the values within the raster pointed to by gdobj specified by
    the row, col values in rc. Sets nodata. Returns numpy array.

    Parameters
    ----------
    gdobj : [type]
        Points to raster to get values from
    rc : [type]
        [description]
    nodata : int, optional
        No data value for the raster, by default -9999

    Returns
    -------
    numpy.ndarray
        Raster values
    """

    vals = []
    for r, c in zip(rc[0], rc[1]):
        vals.append(gdobj.ReadAsArray(xoff=int(c), yoff=int(r), xsize=1, ysize=1)[0][0])
    vals = np.array(vals)
    vals[vals == nodata] = np.nan

    return vals

# def pts_to_line_segments(pts):
#     """ Converts a list of shapely points to a set of line segments. Points
#     should be in order.

#     Parameters
#     ----------
#     pts : list of shapely.geometry.Point
#         [description]

#     Returns
#     -------
#     list of shapely.geometry.LineString
#         length N-1, where N=length(pts)
#     """
#     ls = [LineString((pts[i], pts[i + 1])) for i in range(len(pts) - 1)]

#     return ls

# def find_nangroups(arr):
#     """Returns groups of nans in an array.

#     Parameters
#     ----------
#     arr : [type]
#         [description]

#     Returns
#     -------
#     list
#         [description]
#     """
#     nans = np.isnan(arr)
#     nangroups = []
#     nangroup = []
#     for i, n in enumerate(nans):
#         if not n:
#             if len(nangroup) > 0:
#                 nangroups.append(nangroup)
#             nangroup = []
#         else:
#             nangroup.append(i)

#     return nangroup

# def interpolate_nangroups(arr, dists, nangroups):
#     """Linearly interpolates across groups of nans in a 1-D array.

#     Parameters
#     ----------
#     arr : [type]
#         [description]
#     dists : [type]
#         [description]
#     nangroups : [type]
#         [description]

#     Returns
#     -------
#     [type]
#         [description]
#     """
#     for ng in nangroups:
#         if type(ng) is int:
#             ng = [ng]
#         if 0 in ng or len(arr) - 1 in ng:
#             continue
#         interp = si.interp1d(
#             (dists[ng[0] - 1], dists[ng[-1] + 1]), (arr[ng[0] - 1], arr[ng[-1] + 1])
#         )
#         interp_pts = dists[np.arange(ng[0], ng[-1] + 1)]
#         arr[np.arange(ng[0], ng[-1] + 1)] = interp(interp_pts)

#     return arr

max_stars_repo_name: jonschwenk/rabpro

max_issues_repo_name: jonschwenk/rabpro

max_forks_repo_name: jonschwenk/rabpro

reason: import numpy,import scipy


text:
import matplotlib.pyplot as plt
import numpy as np

def feval(funcName, *args):
    return eval(funcName)(*args)

def backwardEuler(func, yinit, x_range, h):
    m = len(yinit)
    n = int((x_range[-1] - x_range[0])/h)

    x = x_range[0]
    y = yinit

    xsol = np.empty(0)
    xsol = np.append(xsol, x)

    ysol = np.empty(0)
    ysol = np.append(ysol, y)

    for i in range(n):
        yprime = feval(func, x+h, y)/(1+h)

        for j in range(m):
            y[j] = y[j] + h*yprime[j]

        x += h
        xsol = np.append(xsol, x)

        for r in range(len(y)):
            ysol = np.append(ysol, y[r])  # Saves all new y's

    return [xsol, ysol]

def myFunc(x, y):
    '''
    We define our ODEs in this function.
    '''
    dy = np.zeros((len(y)))
    dy[0] = 3*(1+x) - y[0]
    return dy

h = 0.2
x = np.array([1.0, 2.0])
yinit = np.array([4.0])

[ts, ys] = backwardEuler('myFunc', yinit, x, h)

# Calculates the exact solution, for comparison
dt = int((x[-1] - x[0]) / h)
t = [x[0]+i*h for i in range(dt+1)]
yexact = []
for i in range(dt+1):
    ye = 3 * t[i] + np.exp(1 - t[i])
    yexact.append(ye)

plt.plot(ts, ys, 'r')
plt.plot(t, yexact, 'b')
plt.xlim(x[0], x[1])
plt.legend(["Backward Euler method",
            "Exact solution manually computed"], loc = 2)
plt.xlabel('x', fontsize = 10)
plt.ylabel('y', fontsize = 10)
plt.tight_layout()
plt.show()

max_stars_repo_name: brtymn/python-mini-projects

max_issues_repo_name: brtymn/python-mini-projects

max_forks_repo_name: brtymn/python-mini-projects


text:
import numpy as np
import random
import math
import numbers

def generate_matrix(m, d):
	F = np.zeros((m,d))

	bound = -1 / math.sqrt(m)
	for i in range(m):
		for j in range(d):
			F[i][j] = random.uniform(-bound, bound)

	return F
generate_matrix(5, 10)

class Random_Matrix_client():
	def __init__(self, F, m, d, e):
		# initialization of the protocol's constants
		self.F = F
		self.m = m
		self.d = d
		self.e = e

	def encode(self, v):
		r = random.randint(0, self.m - 1)
		x = self.F[r][v]

		return (r, x)

	def perturbe(self, ret):
		r, x = ret

		pr = math.exp(self.e) / (math.exp(self.e) + 1)
		res = random.random()
		if (res < pr):
			b = 1
		else:
			b = -1

		c = (math.exp(self.e) + 1) / (math.exp(self.e) - 1)

		return (r, b * c * self.m * x)

	def randomize(self, v):
		return self.perturbe(self.encode(v))

class Random_Matrix_aggregator():
	def __init__(self, F, m, d, e):
		# initialization of the protocol's constants
		self.F = F
		self.m = m
		self.d = d
		self.e = e

	def aggregate(self, config):

		reported_values = config['reported_values']
		public_matrix = config['public_matrix']
		d = self.d

		results = np.zeros(d)
		for i in range(d):
			sum_v = 0
			for j in reported_values:
				sum_v += j[1] * public_matrix[j[0]][i]

			results[i] = sum_v
		return results

max_stars_repo_name: nikosgalanis/bsc-thesis

max_issues_repo_name: nikosgalanis/bsc-thesis

max_forks_repo_name: nikosgalanis/bsc-thesis


text:
import numpy as np
import pandas as pd
from scipy.linalg import block_diag, norm
from pyFBS.utility import coh_frf

class VPT(object):
    """
    Virtual Point Transformation (VPT) - enables transformation of measured responses to a virtual DoFs.  Current
    implementation enables rigid interface deformation modes and all 6 DoFs (3 translations + 3 rotations), have to
    be included in the transformation.

    :param ch: A DataFrame containing information on channels (i.e. outputs)
    :type ch: pd.DataFrame
    :param refch: A DataFrame containing information on reference channels (i.e. inputs)
    :type refch: pd.DataFrame
    :param vp_ch: A DataFrame containing information on virtual point channels
    :type vp_ch: pd.DataFrame
    :param vp_refch: A DataFrame containing information on reference virtual point channels
    :type vp_refch: pd.DataFrame
    :param Wu: Displacement weigting matrix
    :type Wu: array(float), optional
    :param Wf: Force weighting matrix
    :type Wf: array(float), optional
    :param sort_matrix: Sort transformation matrixes
    :type sort_matrix: bool, optional
    """

    def __init__(self, ch, refch, vp_ch, vp_refch, Wu = None, Wf = None, sort_matrix = True):
        self.sort_matrix = sort_matrix

        # Load the physical input-output DoFs
        self.Channels = ch
        self.RefChannels = refch

        # Load virtual input-output DoFs
        self.Virtual_Channels = vp_ch
        self.Virtual_RefChannels = vp_refch

        # Load Weighting matrices if None, no weighting is applied in the transformation
        self.Wu_p = Wu
        self.Wf_p = Wf

        # Define the IDM_U and IDM_F matrix
        self.define_IDM_U()
        self.define_IDM_F()

    def define_IDM_U(self):
        """
        Calculates Ru, Tu and Fu matrices based on the supplied position and orientation of Channels and Virtual
        Channels.
        """
        ov_u, _vps, mask_u = self.find_overlap_ch(self.Channels, self.Virtual_Channels)

        R_all = []

        # iterates through all unique virtual points (through grouping)
        _Warray = []
        for i in range(len(ov_u)):
            # gets the unique VP position
            _posVP = np.asarray(self.Virtual_Channels.iloc[i][["Position_1","Position_2","Position_3"]].to_numpy())
            # gets the unique VP orientation
            _dirVP = np.asarray(self.Virtual_Channels.iloc[i:i+3][["Direction_1","Direction_2","Direction_3"]].to_numpy())
            # gets the current positions
            ov_c = ov_u[i]
            # gets defined DoF for specific VP
            _desc = self.Virtual_Channels["Description"].to_list()

            r = np.zeros((len(ov_c[0]), len(_desc)))
            for j, ch in enumerate(ov_c[0]):
                _pos = np.asarray(self.Channels.iloc[ch][["Position_1","Position_2","Position_3"]].to_numpy()).astype(float)
                _dir = np.asarray(self.Channels.iloc[ch][["Direction_1","Direction_2","Direction_3"]].to_numpy()).astype(float)
                _group = self.Channels.iloc[ch]["Grouping"]
                _type = self.Channels.iloc[ch]["Quantity"]
                r[j, :] = (_dirVP @ _dir) @ self.R_matrix_U(_posVP - _pos, _desc, type=_type)
                _Warray.append(self.W_rotational(_pos, _dir, type=_type))
            R_all.append(r)

        Ru = block_diag(*R_all, np.eye(len(np.where(mask_u != 0)[0])))

        if self.sort_matrix:
            R_n = np.zeros_like(Ru)
            # position the transformation matrix based on location it the .xlsx file
            R_all = np.asarray(R_all)[0, :, :]
            gg = 0
            trig = True
            for i, k in enumerate(mask_u):
                if k == 1:
                    R_n[i, gg] = 1
                    gg += k
                else:
                    if trig:
                        R_n[i:i + R_all.shape[0], gg:gg + R_all.shape[1]] = R_all
                        gg += R_all.shape[1]
                        trig = False
            Ru = R_n

        # definition of weighting matrix
        if self.Wu_p == None:
            Wu = block_diag(*_Warray)
            Wu = block_diag(Wu, np.eye(len(np.where(mask_u != 0)[0])))
        else:
            Wu = block_diag(*self.Wu_p)
            Wu = block_diag(Wu, np.eye(len(np.where(mask_u != 0)[0])))

        # calculate the Tu, Fu matrices
        Tu = np.linalg.pinv(Ru.T @ Wu @ Ru) @ Ru.T @ Wu
        Fu = Ru @ Tu

        self.Ru = Ru
        self.Wu = Wu
        self.Tu = Tu
        self.Fu = Fu

    def define_IDM_F(self):
        """
        Calculates the Rf, Tf, Ff matrices based on the supplied position and orientation of Reference Channels and
        Reference Virtual Channels.
        """

        ov_f, _vps, mask_f = self.find_overlap_ch(self.RefChannels, self.Virtual_RefChannels)
        # print(mask_f)
        R_all = []
        # iterates through all unique virtual points (through grouping)
        for i in range(len(ov_f)):
            # gets the unique VP position
            _posVP = np.asarray(self.Virtual_RefChannels.iloc[i][["Position_1","Position_2","Position_3"]].to_numpy())
            # gets the unique VP orientation
            _dirVP = np.asarray(self.Virtual_RefChannels.iloc[i:i+3][["Direction_1","Direction_2","Direction_3"]].to_numpy())
            # gets the current positions
            ov_c = ov_f[i]
            # gets defined DoF for specific VP
            _desc = self.Virtual_RefChannels["Description"].to_list()

            r = np.zeros((len(ov_c[0]), len(_desc)))
            for j, ch in enumerate(ov_c[0]):
                _pos = np.asarray(self.RefChannels.iloc[ch][["Position_1", "Position_2", "Position_3"]].to_numpy()).astype(float)
                _dir = np.asarray(self.RefChannels.iloc[ch][["Direction_1", "Direction_2", "Direction_3"]].to_numpy()).astype(float)
                _group = self.RefChannels.iloc[ch]["Grouping"]
                _type = self.RefChannels.iloc[ch]["Quantity"]
                r[j, :] = (self.R_matrix_F(_posVP - _pos, _desc) @ (_dirVP @ _dir).T).reshape(-1)
            R_all.append(r)

        # position the transformation matrix based on location it the .xlsx file
        Rf = block_diag(*R_all, np.eye(len(np.where(mask_f != 0)[0])))

        if self.sort_matrix:
            R_n = np.zeros_like(Rf)
            R_all = np.asarray(R_all)[0, :, :]
            gg = 0
            trig = True
            for i, k in enumerate(mask_f):
                if k == 1:
                    R_n[i, gg] = 1
                    gg += k
                else:
                    if trig:
                        R_n[i:i + R_all.shape[0], gg:gg + R_all.shape[1]] = R_all

                        gg += R_all.shape[1]
                        trig = False
            Rf = R_n

        # definition of weighting matrix
        if self.Wf_p == None:
            Wf = np.eye(np.max(Rf.shape))
        else:
            Wf = self.Wf_p

        # calculate the Tf, Ff matrices
        Tf = Wf @ Rf @ np.linalg.pinv(Rf.T @ Wf @ Rf)
        Ff = Rf @ Tf.T

        self.Rf = Rf
        self.Wf = Wf
        self.Tf = Tf
        self.Ff = Ff

    @staticmethod
    def R_matrix_U(pos, desc, type="Acceleration"):
        """
        Calculate Ru matrix based on the channel position/orientation and sensor type.

        :param pos: Position of the channel.
        :type pos: array(float)
        :param type: Type of the channel (i.e. Acceleration or Angular Acceleration).
        :type pos: string, optional
        :returns: Ru matrix
        """

        rx, ry, rz = pos[0], pos[1], pos[2]

        _R = np.asarray([[1, 0, 0, 0, rz, -ry],
                         [0, 1, 0, -rz, 0, rx],
                         [0, 0, 1, ry, -rx, 0]])

        if type == "Angular Acceleration":
            _R = np.asarray([[0, 0, 0, 1, 0, 0],
                             [0, 0, 0, 0, 1, 0],
                             [0, 0, 0, 0, 0, 1]])

        # isolating desired DoF
        _R = np.asarray(pd.DataFrame(_R, columns=['ux','uy','uz','tx','ty','tz'])[desc])

        return _R

    @staticmethod
    def W_rotational(pos, dir, type="Angular Acceleration"):
        """
        Defines the weighting matrix based on the location of rotational accelerometer

        :param pos: Position of the channel.
        :type pos: array(float)
        :param dir: Direction of the channel
        :type dir: array(float)
        :param type: Type of the channel (i.e. Acceleration or Angular Acceleration)
        :type type: str, optional
        """

        rx, ry, rz = pos[0], pos[1], pos[2]

        _W = 1

        if type == "Angular Acceleration":
            c = np.where(np.asarray(dir) != 0)[1][0]
            if c == 0:
                _W = np.sqrt(rz ** 2 + ry ** 2) ** 2
            elif c == 1:
                _W = np.sqrt(rz ** 2 + rx ** 2) ** 2
            elif c == 2:
                _W = np.sqrt(ry ** 2 + rx ** 2) ** 2

        return _W

    @staticmethod
    def R_matrix_F(pos, desc):
        """
        Calculates Rf matrix based on the reference channel position/orientation.

        :param pos: Position of the reference channel relative to the virtual point.
        :type pos: array(float)
        :returns: Rf matrix
        """

        rx, ry, rz = pos[0], pos[1], pos[2]

        _R = np.asarray([[1, 0, 0],
                         [0, 1, 0],
                         [0, 0, 1],
                         [0, -rz, ry],
                         [rz, 0, -rx],
                         [-ry, rx, 0]])

        # isolating desired DoF
        _R = np.asarray(pd.DataFrame(_R.T, columns=['fx','fy','fz','mx','my','mz'])[desc]).T

        return _R

    @staticmethod
    def find_overlap_ch(channelsA, channelsB):
        """
        Finds an overlap of grouping number between two channel DataFrames.

        :param channelsA: First set of channels
        :type channelsA: pd.DataFrame
        :param channelsB: Second set of channels
        :type channelsB: pd.DataFrame
        :return: overlap,unique_index, overlap_mask
        """

        # Get the grouping numbers from DataFrames
        _group_ch = channelsA.Grouping.to_numpy()
        _group_chVP = channelsB.Grouping.to_numpy()

        # Find overlap between the two channel datasets
        _overlap = []
        for a in np.unique(_group_chVP):
            _overlap.append(np.where(_group_ch == a))

        mask = np.isin(_group_ch, np.unique(_group_chVP), invert=True).astype(int)

        return _overlap, np.unique(_group_chVP, return_index=True), mask

    @staticmethod
    def find_group(gr, gr_list):
        """
        Get a grouping overlap between two DataFrames.

        :param gr: Grouping number
        :type gr: int
        :param gr_list: A list of grouping numbers
        :type gr_list: list
        :return: overlap_mask
        """

        _overlap = []
        for a in np.unique(gr):
            _overlap.append(np.where(gr_list == a))
        return np.asarray(_overlap).reshape(-1)

    def apply_VPT(self, freq,FRF):
        """
        Applies the Virtual Point Transformation on the FRF matrix.

        :param freq: Frequency vector
        :type freq: array(float)
        :param FRF: A matrix of Frequency Response Functions FRFs [f,out,in].
        :type FRF: array(float)
        """

        _Y_vpt = self.Tu @ FRF @ self.Tf

        self.vptData = _Y_vpt
        self.freq = freq
        self.FRF = FRF

    def consistency(self, grouping, ref_grouping):
        """
        Evaluates VP consistency indicators based on the supplied grouping numbers.

        :param grouping: Grouping number of the VP.
        :type grouping: float
        :param ref_grouping: Grouping number of the reference VP.
        :type ref_grouping: float
        """

        # get all groupings from the vpt
        _ch_all = self.Channels.Grouping.to_numpy()
        _chVP_all = self.Virtual_Channels.Grouping.to_numpy()

        _Rch_all = self.RefChannels.Grouping.to_numpy()
        _RchVP_all = self.Virtual_RefChannels.Grouping.to_numpy()

        # extract the grouping mask
        ind_ch = self.find_group(grouping, _ch_all)
        ind_Rch = self.find_group(ref_grouping, _Rch_all)

        # Calculate sensor consistency
        sub_Y = np.transpose(self.FRF,(1,2,0))[ind_ch, :, :][:, ind_Rch, :]
        sub_Fu = self.Fu[ind_ch, :][:, ind_ch]

        u_f = np.zeros((sub_Y.shape[0], 1, sub_Y.shape[2]), dtype=complex)
        u = np.zeros((sub_Y.shape[0], 1, sub_Y.shape[2]), dtype=complex)

        for i in range(sub_Y.shape[2]):
            # filtered response
            u_f[:, :, i] = sub_Fu @ sub_Y[:, :, i] @ np.ones((sub_Y.shape[1], 1))
            # initial response
            u[:, :, i] = sub_Y[:, :, i] @ np.ones((sub_Y.shape[1], 1))

        self.u_f = u_f[:,0,:]
        self.u = u[:,0,:]

        # Calculate overall sensor consistency indicator
        self.overall_sensor = norm(self.u_f,axis = 0) / norm(self.u,axis = 0)

        # Calculate specific sensor consistency indicator
        specific_sensor = []
        for i in range(self.u.shape[0]):
            specific_sensor.append(coh_frf(self.u_f[i, :], self.u[i, :]))

        self.specific_sensor = np.asarray(specific_sensor)

        # Calculate impact consistency
        sub_Y = np.transpose(self.FRF,(1,2,0))[ind_ch, :, :][:, ind_Rch, :]
        sub_Ff = self.Ff[ind_Rch, :][:, ind_Rch]

        y_f = np.zeros((sub_Y.shape[1], 1, sub_Y.shape[2]), dtype=complex)
        y = np.zeros((sub_Y.shape[1], 1, sub_Y.shape[2]), dtype=complex)

        for i in range(sub_Y.shape[2]):
            # filtered response
            y_f[:, :, i] = (np.ones((sub_Y.shape[0], 1)).T @ sub_Y[:, :, i] @ sub_Ff).T
            # initial response
            y[:, :, i] = (np.ones((sub_Y.shape[0], 1)).T @ sub_Y[:, :, i]).T

        self.y_f = y_f[:,0,:]
        self.y = y[:,0,:]

        # Calculate overall impact consistency indicator
        self.overall_impact = norm(self.y_f,axis = 0) / norm(self.y,axis = 0)

        # Calculate specific impact consistency indicator
        specific_impact = []
        for i in range(self.y.shape[0]):
            specific_impact.append(coh_frf(self.y_f[i,:],self.y[i,:]))

        self.specific_impact = np.asarray(specific_impact)

    """
    Frequency-dependend weighting matrix - to be implemented in the pyFBS with next release

    Wu = block_diag(*_Warray)
    Wu = block_diag(Wu, np.eye(len(np.where(mask_u != 0)[0])))
    self.Wu = Wu

    # Wu_f is a 4D numpy array for each input set you use where -j refers to the freq input W[i,:,:,j]
    # n_imp... number of impacts
    # n_out... number of outputs
    # n_freq.. number of freqs

    n_out = self.Y.Data.shape[0]
    n_imp = self.Y.Data.shape[1]
    n_freq = self.Y.Data.shape[2]

    Wu_f = np.zeros((n_imp, n_out, n_freq))
    Tu_f = np.zeros((n_imp, self.Ru.shape[1], n_out, n_freq))
    # Fu_f = np.zeros((n_imp, n_out , n_out , 1000))

    for _f in tqdm(range(n_freq)):
        for _i in range(n_imp):
            _tW = block_diag(*np.abs(self.Y.Coherence[:, _i, _f])) ** 2
            # print(_tW.shape)
            # for _d,diag_val in enumerate(np.diag(_tW)):
            #    if _d in [9,10,11,21,22,23,33,34,35]:
            #        _tW[_d,_d] = 0#sigmoid(self.Y.Freqs[_f])

            W_sum = Wu + _tW
            Wu_f[_i, :, _f] = np.diag(W_sum)

            Tu = np.linalg.pinv(Ru.T @ W_sum @ Ru) @ Ru.T @ W_sum
            # Fu = Ru @ Tu
            Tu_f[_i, :, :, _f] = Tu
            # Fu_f[_i, :, :, _f] = Fu

    self.Tu = Tu
    self.Wu_f = Wu_f

    self.Tu_f = Tu_f
    self.Fu_f = Fu_f
    """

reason: import numpy,from scipy


text:
from pyecharts import options as opts
from pyecharts.charts import Geo
from pyecharts.globals import ChartType
from pyecharts.render import make_snapshot
from snapshot_phantomjs import snapshot
import pandas as pd
import numpy as np

df = pd.read_csv("./weather.csv", encoding='utf-8', dtype=np.str)
df = pd.DataFrame(df, columns=['temperature'])
df['temperature'] = df['temperature'].astype(np.float)
day = 0

class Data:
    guangdong_city = ["深圳市"]

    @staticmethod
    def values() -> list:
        res = []
        for i in range(len(df)):
            res.append(float(df['temperature'][i]))
        return res

def geo_guangdong(title) -> Geo:
    c = (
        Geo()
        .add_schema(maptype="广东")
        .add(
            title,
            [list(z) for z in zip(Data.guangdong_city, [Data.values()[day]])],
            type_=ChartType.HEATMAP,
        )
        .set_global_opts(
            visualmap_opts=opts.VisualMapOpts(min_=0, max_=int(df['temperature'].max())),#is_piecewise=True),
            title_opts=opts.TitleOpts(title="广东省深圳市12月13-27日温度变化情况"),
        )
    )
    return c

for i in range(10):
    day = i
    str_date="12月" + str(i+13) + "日"
    make_snapshot(snapshot, geo_guangdong(str_date).render(),
                  str(i+1)+".png", pixel_ratio=1)

max_stars_repo_path: Course 202009/Python/Visualize2/4.py

max_stars_repo_name: Seizzzz/DailyCodes

max_issues_repo_path: Course 202009/Python/Visualize2/4.py

max_issues_repo_name: Seizzzz/DailyCodes

max_forks_repo_path: Course 202009/Python/Visualize2/4.py

max_forks_repo_name: Seizzzz/DailyCodes


text:
# Copyright (c) 2021 Graphcore Ltd. All rights reserved.
import pytest
import time
import numpy as np
import torch
import popart

import test_utils

def torch_ctc_loss(input_data, target_data, input_lengths, target_lengths, opts):
    """For a representative input, calculate the pytorch CTC loss and the input gradients
    to use as reference for IPU implementation"""

    # PyTorch CPU doesn't support FP16
    if opts.precision == "FLOAT16":
        input_data = input_data.astype(np.float32)

    # PyTorch CTCLoss expect inputs with shape (seq_length, batch_size, num_symbols)
    input_data = np.transpose(input_data, (1, 0, 2)).reshape(
        (opts.input_size, opts.batch_size, opts.num_classes)
    )

    # PyTorch doesn't support uint32
    target_data = target_data.astype(np.int32).reshape(
        (opts.batch_size, opts.target_size)
    )
    _, batch_size, _ = input_data.shape
    assert target_data.shape[0] == batch_size

    ctc_loss = torch.nn.CTCLoss(blank=0, reduction=opts.reduction_type)

    t0 = time.perf_counter()
    torch_input_data = torch.from_numpy(input_data).requires_grad_()

    loss = ctc_loss(
        torch_input_data,
        torch.from_numpy(target_data),
        torch.from_numpy(input_lengths.astype(np.int32)),
        torch.from_numpy(target_lengths.astype(np.int32)),
    )
    duration = time.perf_counter() - t0
    print("Execution took %.3f" % duration)
    print("Pytorch loss", loss)

    loss.backward(retain_graph=True)
    return loss.data.numpy(), torch_input_data.grad

def build_popart_graph(opts):
    """Build a popart graph containing only the CTC loss, with specified input sizes.
    Anchors the gradient for comparison with the PyTorch reference."""

    builder = popart.Builder()
    inputs_info = popart.TensorInfo(
        opts.precision, [opts.batch_size, opts.input_size, opts.num_classes]
    )
    inputs = builder.addInputTensor(inputs_info, "log_probs")
    log_probs = builder.aiOnnx.logsoftmax([inputs], axis=2)

    targets_info = popart.TensorInfo("UINT32", [opts.batch_size, opts.target_size])
    targets = builder.addInputTensor(targets_info, "targets")

    input_lengths_info = popart.TensorInfo("UINT32", [opts.batch_size])
    input_lengths = builder.addInputTensor(input_lengths_info, "input_length")

    target_lengths_info = popart.TensorInfo("UINT32", [opts.batch_size])
    target_lengths = builder.addInputTensor(target_lengths_info, "target_length")

    if opts.reduction_type == "mean":
        reduction_type = popart.ReductionType.Mean
    elif opts.reduction_type == "sum":
        reduction_type = popart.ReductionType.Sum
    else:
        reduction_type = popart.ReductionType.NoReduction

    partial32 = 1 if opts.partial32 else 0

    output_tensors = builder.customOp(
        opName="CtcLoss",
        opVersion=1,
        domain="com.acme",
        inputs=[log_probs, targets, input_lengths, target_lengths],
        attributes={
            "blank": 0,
            "reduction": int(reduction_type),
            "partial32": partial32,
        },
        numOutputs=4,
    )
    neg_log_likelihood = output_tensors[0]
    proto = builder.getModelProto()
    gradient = popart.reservedGradientPrefix() + log_probs
    anchors = {
        gradient: popart.AnchorReturnType("ALL"),
        neg_log_likelihood: popart.AnchorReturnType("ALL"),
        log_probs: popart.AnchorReturnType("ALL"),
    }
    data_flow = popart.DataFlow(1, anchors)
    if opts.ipu:
        device = popart.DeviceManager().acquireAvailableDevice(1)
    else:
        device = popart.DeviceManager().createIpuModelDevice({"tilesPerIPU": 1216})
    optimizer = popart.ConstSGD(0.001)
    options = popart.SessionOptions()
    options.enableFloatingPointChecks = False
    session = popart.TrainingSession(
        fnModel=proto,
        dataFlow=data_flow,
        loss=neg_log_likelihood,
        optimizer=optimizer,
        deviceInfo=device,
        userOptions=options,
    )
    t0 = time.perf_counter()
    session.prepareDevice()
    result = session.initAnchorArrays()
    session.weightsFromHost()

    duration = time.perf_counter() - t0
    print("Device prep took %.3f" % duration)
    return (
        session,
        inputs,
        targets,
        input_lengths,
        target_lengths,
        result,
        log_probs,
        gradient,
        neg_log_likelihood,
    )

def run_popart_graph(context, input_data, target_data, input_length_data, target_length_data):
    """ Runs the popart graph for the given inputs """
    (
        session,
        inputs,
        targets,
        input_lengths,
        target_lengths,
        result,
        log_probs,
        gradient,
        neg_log_likelihood,
    ) = context
    data = {
        inputs: input_data,
        targets: target_data,
        input_lengths: input_length_data,
        target_lengths: target_length_data,
    }

    t0 = time.perf_counter()
    stepio = popart.PyStepIO(data, result)
    session.run(stepio)
    duration = time.perf_counter() - t0
    print("Execution took %.3f" % duration)
    print("Nllloss IPU", result[neg_log_likelihood])
    return (result, log_probs, gradient, neg_log_likelihood)

def build_and_run_popart_graph(input_data, target_data, input_length_data, target_length_data, opts):
    """ Build and run a popart graph containing only the CTC loss, with specified input tensor data.
    Runs a single training step and anchors the gradient for comparison with the PyTorch reference. """

    context = build_popart_graph(opts)

    return run_popart_graph(
        context, input_data, target_data, input_length_data, target_length_data
    )

def run_single_case(args):
    """ Run a single CTC loss test case """
    np.random.seed(42)

    (inputs, target, input_length_data, target_length_data) = test_utils.generate_data(args)
    (result, log_probs, gradients, nll) = build_and_run_popart_graph(
        inputs, target, input_length_data, target_length_data, args
    )
    ipu_grad = result[gradients]
    ipu_loss = result[nll]
    print(ipu_grad)

    pytorch_loss, pytorch_res = torch_ctc_loss(
        result[log_probs], target, input_length_data, target_length_data, args
    )

    ipu_grad = np.transpose(ipu_grad, (1, 0, 2)).reshape(pytorch_res.shape)

    grad_err = test_utils.getTensorError(ipu_grad, pytorch_res)
    loss_err = test_utils.getTensorError(ipu_loss, pytorch_loss)
    print("Gradient Error", grad_err)
    print("Loss Error", loss_err)

    print(f"IPU Loss: {ipu_loss}")
    print("Grad result: " + ("Pass" if grad_err < 1e-4 else "FAIL"))
    print("Loss result: " + ("Pass" if loss_err < 1e-5 else "FAIL"))
    return grad_err, loss_err

# Note that we do not test NoReduction at BS > 1 as Popart requires scalar losses
@pytest.mark.ipus(1)
@pytest.mark.category1
@pytest.mark.parametrize(
    "input_size, target_size, num_classes, batch_size, reduction_type, ipu",
    [
        (4, 2, 5, 1, "mean", False),
        (4, 2, 5, 1, "mean", True),
        (4, 2, 5, 1, "sum", False),
        (4, 2, 5, 1, "sum", True),
        (4, 2, 5, 1, "none", False),
        (4, 2, 5, 1, "none", True),
        (4, 2, 5, 2, "mean", False),
        (4, 2, 5, 2, "mean", True),
        (4, 2, 5, 2, "sum", False),
        (4, 2, 5, 2, "sum", True),
        (4, 2, 5, 3, "mean", False),
        (4, 2, 5, 3, "mean", True),
        (4, 2, 5, 3, "sum", False),
        (4, 2, 5, 3, "sum", True),
    ],
)
def test_ctc_loss_batch_size(custom_ops, input_size, target_size, num_classes, batch_size, reduction_type, ipu):
    """ test across different batch-sizes """
    args = test_utils.args_from_params(
        input_size, target_size, num_classes, batch_size, reduction_type
    )
    grad_err, loss_err = run_single_case(args)
    assert grad_err < 1e-4
    assert loss_err < 1e-5

@pytest.mark.ipus(1)
@pytest.mark.category1
@pytest.mark.parametrize(
    "reduction_type, precision, ipu",
    [("mean", "FLOAT", False), ("mean", "FLOAT16", False), ("sum", "FLOAT", False), ("sum", "FLOAT16", False),
     ("mean", "FLOAT", True), ("mean", "FLOAT16", True), ("sum", "FLOAT", True), ("sum", "FLOAT16", True)],
)
def test_ctc_loss_precision(custom_ops, reduction_type, precision, ipu):
    """ test across different reduction types and precisions """
    params = {
        "input_size": 6,
        "target_size": 3,
        "num_classes": 5,
        "batch_size": 3,
        "reduction_type": reduction_type,
        "precision": precision,
        "ipu": ipu,
    }

    args = test_utils.args_from_params(**params)
    grad_err, loss_err = run_single_case(args)
    assert grad_err < 1e-4
    assert loss_err < 1e-5

@pytest.mark.ipus(1)
@pytest.mark.category1
@pytest.mark.parametrize(
    "batch_size, precision, ipu",
    [(1, "FLOAT16", False), (1, "FLOAT", False), (2, "FLOAT16", False), (2, "FLOAT", False),
     (4, "FLOAT16", False), (4, "FLOAT", False),
     (1, "FLOAT16", True), (1, "FLOAT", True), (2, "FLOAT16", True), (2, "FLOAT", True),
     (4, "FLOAT16", True), (4, "FLOAT", True)
     ],
)
def test_ctc_loss_asr_dim(custom_ops, batch_size, precision, ipu):
    """ test for typical ASR model dimensions"""
    num_classes = 36
    input_size = 375
    target_size = 200
    reduction_type = "mean"

    args = test_utils.args_from_params(
        input_size, target_size, num_classes, batch_size, reduction_type, precision, ipu
    )
    grad_err, loss_err = run_single_case(args)
    assert grad_err < 1e-4
    assert loss_err < 1e-5

@pytest.mark.ipus(1)
@pytest.mark.category1
@pytest.mark.parametrize(
    "reduction_type, variable_input, ipu",
    [("mean", True, False), ("mean", False, False), ("sum", True, False), ("mean", False, False),
     ("mean", True, True), ("mean", False, True), ("sum", True, True), ("mean", False, True)],
)
def test_ctc_loss_variable_tgt(custom_ops, reduction_type, variable_input, ipu):
    """ test for both constant and variable sequence lengths """
    params = {
        "input_size": 20,
        "target_size": 10,
        "num_classes": 5,
        "batch_size": 3,
        "reduction_type": reduction_type,
        "variable_input": variable_input,
        "ipu": ipu,
    }

    args = test_utils.args_from_params(**params)
    grad_err, loss_err = run_single_case(args)
    assert grad_err < 1e-4
    assert loss_err < 1e-5

if __name__ == "__main__":

    test_utils.load_custom_ops_lib()
    args = test_utils.parse_args()
    run_single_case(args)


text:
#!/usr/bin/python
'''
Program:
    This code is used to find darks for images
Usage:
    find_dark.py
Editor:
    Jacob975
20180621
#################################
update log
20180621 alaph 6
    1. The code work
'''
import os
from astropy.io import fits as pyfits
import numpy as np
import glob
import time
import fnmatch
import TAT_env
import datetime
from datetime import datetime, timedelta

def match_date(current_date, date_list):
    # comfirm the type of date and datelist is int.
    current_date = int(current_date)
    new_date_list = []
    for d in date_list:
        try :
            int(d)
        except:
            continue
        else:
            new_date_list.append(int(d))
    date_list = new_date_list
    # get all delta between two dates.
    delta_list = []
    for date in date_list:
        delta_list.append(abs(date-current_date))
    # find minimum and index of minimum.
    min_delta = min(delta_list)
    index = delta_list.index(min_delta)
    nearest_date = str(date_list[index])
    return nearest_date

def stack_mdn_method(name_list):
    data_list = []
    # Use the first 20 image only, or it would be out of buffer.
    for name in name_list[:20]:
        data = pyfits.getdata(name)
        data_list.append(data)
    data_list = np.array(data_list)
    mdn_data = np.median(data_list, axis = 0)
    return mdn_data

def get_dark_to(site, exptime, date, path_of_dark):
    temp_path=os.getcwd()
    # next date
    next_date = (datetime.strptime(date, '%Y%m%d') + timedelta(days=1)).strftime('%Y%m%d')
    # setup keywords of darks
    dark_keywords_1 = "dark{0}{1}*{2}.fit".format(site, date[2:], exptime)
    dark_keywords_2 = "dark{0}{1}*{2}.fit".format(site, next_date[2:], exptime)
    # copy files matching the keywords.
    command = "cp {0} {1} 2>/dev/null".format(dark_keywords_1, path_of_dark)
    os.system(command)
    command = "cp {0} {1} 2>/dev/null".format(dark_keywords_2, path_of_dark)
    os.system(command)
    answer = len(glob.glob("{0}/*.fit".format(path_of_dark)))
    return answer

def sub_process(path, site, date, date_list, path_of_dark, exptime):
    # find another proper date to find dark.
    nearest_date = match_date(date , date_list)
    os.chdir(nearest_date)
    os.system("pwd")
    # get dark
    number = get_dark_to(site, exptime, nearest_date, path_of_dark)
    os.chdir("..")
    date_list.remove(nearest_date)
    return number, nearest_date

#--------------------------------------------
# main code
if __name__ == "__main__":
    # Measure time
    start_time = time.time()
    #----------------------------------------
    image_list = glob.glob("*.fit")
    if len(image_list) == 0:
        print "No image found, find_dark.py stop."
        exit(1)
    # Initialize
    # get info from path
    path=os.getcwd()
    path_of_data = TAT_env.path_of_data
    print "--- Find dark for {0} ---".format(path)
    temp_path = path.split(path_of_data)
    temp_path_2 = temp_path[1].split("/")
    date = temp_path_2[3]
    site = temp_path_2[1]
    # get exptime from header of one of images
    image_list = glob.glob("*.fit")
    darkh = pyfits.getheader(image_list[0])
    exptime = int(darkh["EXPTIME"])
    #----------------------------------------
    # Go to the dir of calibrate
    path_of_calibrate = path_of_data+"/"+site+"/calibrate"
    os.chdir(path_of_calibrate)
    # get a list of all items in calibrate
    date_list = os.listdir(path_of_calibrate)
    # find the nearest date reference to original date.
    nearest_date = match_date(date, date_list)
    # defind the dir of median dark, if it doesn't exist , create it
    path_of_dark = "{0}/{1}/calibrate/{2}/dark_{3}".format(path_of_data, site, nearest_date, exptime)
    print "path of dark is :"+path_of_dark
    if os.path.isdir(path_of_dark)==False:
        temp="mkdir -p "+path_of_dark
        os.system(temp)
    #-----------------------------------------
    # Let's find the proper dark
    number = len(glob.glob(path_of_dark))
    # check whether the number of dark is enough, if no, find other darks.
    while number < 10:
        number, nearest_date = sub_process(path, site, date, date_list, path_of_dark, exptime)
        if len(date_list) == 1:
            print "No enough dark found, find_dark.py stops."
            exit(1)
    os.chdir(path_of_dark)
    dark_list = glob.glob('dark*.fit')
    mdn_dark = stack_mdn_method(dark_list)
    Median_dark_name="Median_dark_{0}_{1}.fits".format(date, exptime)
    print ("The dark name is :" + Median_dark_name)
    pyfits.writeto(Median_dark_name, mdn_dark, overwrite= True)
    command="cp -R {0} {1}".format(Median_dark_name, path)
    os.system(command)
    #---------------------------------------
    # Measure time
    elapsed_time = time.time() - start_time
    print "Exiting Main Program, spending ", elapsed_time, "seconds."

reason: import numpy,from astropy


text:
""" This module is the main data generation module

It containts all the primitives for generating the data that can
later on be used to construct more complex synthetic data structures

"""

import numpy as np
import pandas as pd

from hysynth.utils.hybrid_system.library import construct_variable_name as get_var

def sloped_line_df(line_slope, start=0, num_points=1000):
    """ This function generates a sloped line

    The number of points is important because it corresponds to how long the
    duration of the line will be as well as the last value it hits

    """
    x_coords = np.arange(start=0, stop=num_points)
    y_coords = line_slope * x_coords + start
    return pd.DataFrame({get_var(1): y_coords})

def straight_line_df(line_val, num_points=1000):
    """ This function generates a straight line by just repeating a given value """
    return pd.DataFrame({get_var(1): np.repeat(line_val, repeats=num_points)})

def construct_signal_from_run(components_dict, run_list):
    """ This function is used to construct a signal run from two components
        a component dict, which maps a synth data primitive to a string
        a run list which is a list of strings, representing the desired sequence of components
    """
    return pd.concat(map(lambda seg: components_dict[seg], run_list))

if __name__ == "__main__":
    raise RuntimeError("This module should not be run directly!")

max_stars_repo_name: HySynth/HySynth

max_issues_repo_name: HySynth/HySynth

max_forks_repo_name: HySynth/HySynth


text:
#!/usr/bin/env python
# Copied from fftpack.helper by Pearu Peterson, October 2005
""" Test functions for fftpack.helper module
"""

import numpy as np
from numpy.testing import TestCase, run_module_suite, assert_array_almost_equal
from numpy import fft
from numpy import pi

class TestFFTShift(TestCase):

    def test_definition(self):
        x = [0, 1, 2, 3, 4, -4, -3, -2, -1]
        y = [-4, -3, -2, -1, 0, 1, 2, 3, 4]
        assert_array_almost_equal(fft.fftshift(x), y)
        assert_array_almost_equal(fft.ifftshift(y), x)
        x = [0, 1, 2, 3, 4, -5, -4, -3, -2, -1]
        y = [-5, -4, -3, -2, -1, 0, 1, 2, 3, 4]
        assert_array_almost_equal(fft.fftshift(x), y)
        assert_array_almost_equal(fft.ifftshift(y), x)

    def test_inverse(self):
        for n in [1,4,9,100,211]:
            x = np.random.random((n,))
            assert_array_almost_equal(fft.ifftshift(fft.fftshift(x)), x)

    def test_axes_keyword(self):
        freqs = [[ 0,  1,  2], [ 3,  4, -4], [-3, -2, -1]]
        shifted = [[-1, -3, -2], [ 2,  0,  1], [-4,  3,  4]]
        assert_array_almost_equal(fft.fftshift(freqs, axes=(0, 1)), shifted)
        assert_array_almost_equal(fft.fftshift(freqs, axes=0),
                fft.fftshift(freqs, axes=(0,)))
        assert_array_almost_equal(fft.ifftshift(shifted, axes=(0, 1)), freqs)
        assert_array_almost_equal(fft.ifftshift(shifted, axes=0),
                fft.ifftshift(shifted, axes=(0,)))

class TestFFTFreq(TestCase):

    def test_definition(self):
        x = [0, 1, 2, 3, 4, -4, -3, -2, -1]
        assert_array_almost_equal(9*fft.fftfreq(9), x)
        assert_array_almost_equal(9*pi*fft.fftfreq(9, pi), x)
        x = [0, 1, 2, 3, 4, -5, -4, -3, -2, -1]
        assert_array_almost_equal(10*fft.fftfreq(10), x)
        assert_array_almost_equal(10*pi*fft.fftfreq(10, pi), x)

class TestIRFFTN(TestCase):

    def test_not_last_axis_success(self):
        ar, ai = np.random.random((2, 16, 8, 32))
        a = ar + 1j*ai

        axes = (-2,)

        # Should not raise error
        fft.irfftn(a, axes=axes)

if __name__ == "__main__":
    run_module_suite()

max_stars_repo_path: editing files/Portable Python 3.2.5.1/App/Lib/site-packages/numpy/fft/tests/test_helper.py

reason: import numpy,from numpy


text:
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
#     http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
"""This module contains code to create and manage SageMaker ``LineageTrialComponent``."""
from __future__ import absolute_import

import logging

from typing import List

from sagemaker.apiutils import _base_types
from sagemaker.lineage.query import (
    LineageQuery,
    LineageFilter,
    LineageSourceEnum,
    LineageEntityEnum,
    LineageQueryDirectionEnum,
)
from sagemaker.lineage.artifact import Artifact

LOGGER = logging.getLogger("sagemaker")

class LineageTrialComponent(_base_types.Record):
    """An Amazon SageMaker, lineage trial component, which is part of a SageMaker lineage.

    A trial component is a stage in a trial.
    Trial components are created automatically within the SageMaker runtime and also can be
    created directly. To automatically associate trial components with a trial and experiment
    supply an experiment config when creating a job.
    For example: https://docs.aws.amazon.com/sagemaker/latest/dg/API_CreateTrainingJob.html

    Attributes:
        trial_component_name (str): The name of the trial component. Generated by SageMaker from the
            name of the source job with a suffix specific to the type of source job.
            trial_component_arn (str): The ARN of the trial component.
        display_name (str): The name of the trial component that will appear in UI,
            such as SageMaker Studio.
        source (obj): A TrialComponentSource object with a source_arn attribute.
        status (str): Status of the source job.
        start_time (datetime): When the source job started.
        end_time (datetime): When the source job ended.
        creation_time (datetime): When the source job was created.
        created_by (obj): Contextual info on which account created the trial component.
        last_modified_time (datetime): When the trial component was last modified.
        last_modified_by (obj): Contextual info on which account last modified the trial component.
        parameters (dict): Dictionary of parameters to the source job.
        input_artifacts (dict): Dictionary of input artifacts.
        output_artifacts (dict): Dictionary of output artifacts.
        metrics (obj): Aggregated metrics for the job.
        parameters_to_remove (list): The hyperparameters to remove from the component.
        input_artifacts_to_remove (list): The input artifacts to remove from the component.
        output_artifacts_to_remove (list): The output artifacts to remove from the component.
        tags (List[dict[str, str]]): A list of tags to associate with the trial component.
    """

    trial_component_name = None
    trial_component_arn = None
    display_name = None
    source = None
    status = None
    start_time = None
    end_time = None
    creation_time = None
    created_by = None
    last_modified_time = None
    last_modified_by = None
    parameters = None
    input_artifacts = None
    output_artifacts = None
    metrics = None
    parameters_to_remove = None
    input_artifacts_to_remove = None
    output_artifacts_to_remove = None
    tags = None

    _boto_create_method: str = "create_trial_component"
    _boto_load_method: str = "describe_trial_component"
    _boto_update_method: str = "update_trial_component"
    _boto_delete_method: str = "delete_trial_component"

    _boto_update_members = [
        "trial_component_name",
        "display_name",
        "status",
        "start_time",
        "end_time",
        "parameters",
        "input_artifacts",
        "output_artifacts",
        "parameters_to_remove",
        "input_artifacts_to_remove",
        "output_artifacts_to_remove",
    ]
    _boto_delete_members = ["trial_component_name"]

    @classmethod
    def load(cls, trial_component_name: str, sagemaker_session=None) -> "LineageTrialComponent":
        """Load an existing trial component and return an ``TrialComponent`` object representing it.

        Args:
            trial_component_name (str): Name of the trial component
            sagemaker_session (sagemaker.session.Session): Session object which
                manages interactions with Amazon SageMaker APIs and any other
                AWS services needed. If not specified, one is created using the
                default AWS configuration chain.
        Returns:
            LineageTrialComponent: A SageMaker ``LineageTrialComponent`` object
        """
        trial_component = cls._construct(
            cls._boto_load_method,
            trial_component_name=trial_component_name,
            sagemaker_session=sagemaker_session,
        )
        return trial_component

    def pipeline_execution_arn(self) -> str:
        """Get the ARN for the pipeline execution associated with this trial component (if any).

        Returns:
            str: A pipeline execution ARN.
        """
        trial_component = self.load(
            trial_component_name=self.trial_component_name, sagemaker_session=self.sagemaker_session
        )

        if trial_component.source is None or trial_component.source["SourceArn"] is None:
            return None

        tags = self.sagemaker_session.sagemaker_client.list_tags(
            ResourceArn=trial_component.source["SourceArn"]
        )["Tags"]
        for tag in tags:
            if tag["Key"] == "sagemaker:pipeline-execution-arn":
                return tag["Value"]
        return None

    def dataset_artifacts(
        self, direction: LineageQueryDirectionEnum = LineageQueryDirectionEnum.ASCENDANTS
    ) -> List[Artifact]:
        """Use the lineage query to retrieve datasets that use this trial component.

        Args:
            direction (LineageQueryDirectionEnum, optional): The query direction.

        Returns:
            list of Artifacts: Artifacts representing a dataset.
        """
        query_filter = LineageFilter(
            entities=[LineageEntityEnum.ARTIFACT], sources=[LineageSourceEnum.DATASET]
        )
        query_result = LineageQuery(self.sagemaker_session).query(
            start_arns=[self.trial_component_arn],
            query_filter=query_filter,
            direction=direction,
            include_edges=False,
        )

        return [vertex.to_lineage_object() for vertex in query_result.vertices]

    def models(
        self, direction: LineageQueryDirectionEnum = LineageQueryDirectionEnum.DESCENDANTS
    ) -> List[Artifact]:
        """Use the lineage query to retrieve models that use this trial component.

        Args:
            direction (LineageQueryDirectionEnum, optional): The query direction.

        Returns:
            list of Artifacts: Artifacts representing a dataset.
        """
        query_filter = LineageFilter(
            entities=[LineageEntityEnum.ARTIFACT], sources=[LineageSourceEnum.MODEL]
        )
        query_result = LineageQuery(self.sagemaker_session).query(
            start_arns=[self.trial_component_arn],
            query_filter=query_filter,
            direction=direction,
            include_edges=False,
        )
        return [vertex.to_lineage_object() for vertex in query_result.vertices]

max_stars_repo_name: BonneC/sagemaker-python-sdk

max_issues_repo_name: BonneC/sagemaker-python-sdk

max_forks_repo_name: BonneC/sagemaker-python-sdk

reason: from sage


text:
import cv2
import numpy as np
#ffpyplayer for playing audio
from ffpyplayer.player import MediaPlayer
# change NAME there for each video after ANIM\\
video_path="STAR WARS IX.mp4"
def PlayVideo(video_path):
    video=cv2.VideoCapture(video_path)
    player = MediaPlayer(video_path)
    while True:
        grabbed, frame=video.read()
        audio_frame, val = player.get_frame()
        if not grabbed:
            print("End of video")
            break
        if cv2.waitKey(28) & 0xFF == ord("q"):
            break
        #change NAME here for each video
        cv2.imshow("Star wars 9", frame)
        if val != 'eof' and audio_frame is not None:
            #audio
            img, t = audio_frame
    video.release()
    cv2.destroyAllWindows()
PlayVideo(video_path)


text:
import numpy as np
import matplotlib.pyplot as plt

no_agents = 20
no_bandits = 100

#prob = [0, 1, 2, 3, 4]
prob = [0, 1, 4, 8, 9]
#leg = [str(k*4) for  k in prob]
leg = [str(2*k) for  k in prob]
for i in range(len(leg)):
    leg[i] = "nj = " + leg[i]

Nij_T_s = []
Nij_T = []

#for i in range(0,20,4):
for i in prob:
    Nij_T_s.append(np.load("Com/Total/N_ij_T_s" + str(i*2)+".npy"))
    Nij_T.append(np.load("Com/Total/N_ij_T" + str(i*2)+".npy"))

N_S_Change = [[[[0 for i in range(20000)] for j in range(no_bandits)] for k in range(no_agents)] for t in range(len(prob))]
N_Change   = [[[[0 for i in range(20000)] for j in range(no_bandits)] for k in range(no_agents)] for t in range(len(prob))]

for i in range(len(prob)):
    print(i)
    for j in range(no_agents):
        for k in range(no_bandits):

            N_S_Change[i][j][k] = Nij_T_s[i][j][k][1:20001] - Nij_T_s[i][j][k][0:20000]
            N_Change[i][j][k] = Nij_T[i][j][k][1:20001]- Nij_T[i][j][k][0:20000]
N_Change = np.array(N_Change)
N_S_Change = np.array(N_S_Change)
C_ = []

for i in range(len(prob)):
    c = [[] for a in range(no_agents)]
    for j in range(no_agents):
        for k in range(no_bandits):
            print(i, j, k)
            c[j].append(np.divide(N_Change[i][j][k] - N_S_Change[i][j][k],N_S_Change[i][j][k]))

    C_.append(c)

print(N_S_Change[1][0][44].tolist())
print(N_Change[1][0][44].tolist())
for i in range(len(prob)):
    fig1, ax = plt.subplots(1, 1, figsize=(8,8))
    plt.plot(C_[i][0][99],linewidth=3)
    plt.title("Self")
    plt.show()
    plt.close(fig1)
    fig1, ax = plt.subplots(1, 1, figsize=(8,8))
    plt.plot(C_[i][0][99],linewidth=3)
    plt.title("Total")
    plt.show()
    plt.close(fig1)

print(len(Nij_T_s[0][0][99].tolist()))
print(len(Nij_T_s))
C = []
for i in range(len(prob)):
    c = [[] for a in range(no_agents)]
    for j in range(no_agents):
        for k in range(no_bandits):
            print(i, j, k)
            c[j].append(np.divide(Nij_T[i][j][k] - Nij_T_s[i][j][k], Nij_T_s[i][j][k]))

    C.append(c)

C = np.array(C)
print(C.shape)
for ban in range(no_bandits):
    if ban==99 or ban==44 or ban==3:
        fig1, ax = plt.subplots(1, 1, figsize=(8,8))
        for i in range(len(prob)):
            print(ban, i)
            plt.plot((1/no_agents)*(np.sum(C[i][:,ban], axis=0)),linewidth=3)
            if ban ==99:
                plt.title("Agent's expected Communication Effect for the optimal bandit no " + str(ban))
            if ban ==44:
                plt.title("Agent's expected Communication Effect for the sub optimal bandit no " + str(ban))
            if ban ==3:
                plt.title("Agent's expected Communication Effect for the lowest bandit no " + str(ban))
            plt.xlabel("Time",fontsize=15)
            plt.ylabel("Effect",fontsize=15)
            plt.legend(leg, fontsize=15)

            if ban ==99:
                name = "For_Prob_" + str(i)+ "Agent_Com_Effect_for_"  + "Optimal"
            if ban ==44:
                name = "For_Prob_" + str(i)+ "Agent_Com_Effect_for_"  + "Sub_Optimal"
            if ban ==3:
                name = "For_Prob_" + str(i)+ "Agent_Com_Effect_for_"  + "Lowest_bandit"
            plt.savefig("Com_Effect/Singular/" + name)
        plt.close(fig1)

fig1, ax = plt.subplots(1, 1, figsize=(8,8))
for i in range(len(prob)):

    plt.plot((1/no_agents)*(np.sum(np.sum(C[i][:,0:98], axis=1), axis=0)),linewidth=3)
    plt.title("Agent's Expected Cumulative Communication Effect for all Sub Optimal Bandits")
    plt.xlabel("Time",fontsize=15)
    plt.ylabel("Effect",fontsize=15)
    plt.legend(leg, fontsize=15)
    name = "Cumulative_Com_Effect_for_prob_" + str(i)
    plt.savefig("Com_Effect/AVG/" + name)
plt.close(fig1)


text:
import pandas as pd
import numpy as np
import matplotlib
import seaborn
import sys
sys.path


text:
import numpy as np
from func import (integrate_pendulum)
from param import  Nm, N, dtsymp, qmin, qmax, pmin, pmax
from profit.util.halton import halton

# specify initial conditions (q0, p0)
method = 'all'
# samples = bluenoise(2, N, method, qmin, qmax, pmin, pmax)
samples = halton(N, 2)*np.array([qmax-qmin, pmax-pmin]) + np.array([qmin, pmin])

q = samples[:,0]
p = samples[:,1]*1.5

t0 = 0.0  # starting time
t1 = dtsymp*Nm # end time
t = np.linspace(t0,t1,Nm) # integration points in time

#integrate pendulum to provide data
ysint = integrate_pendulum(q, p, t)

#%%
P = np.empty((N))
Q = np.empty((N))
for ik in range(0, N):
    P[ik] = ysint[ik][1,-1]
    Q[ik] = ysint[ik][0,-1]

zqtrain = Q - q
zptrain = p - P

xtrain = q.flatten()
ytrain = P.flatten()
ztrain1 = zptrain.flatten()
ztrain2 = zqtrain.flatten()
ztrain = np.concatenate((ztrain1, ztrain2))

max_stars_repo_name: krystophny/profit

max_issues_repo_name: krystophny/profit

max_forks_repo_name: krystophny/profit


text:
import numpy as np

from batchopt.optimizers.nuclear_reaction_optimization import (
    NuclearReactionOptimization,
)

def test_nro_init(optimizer_init_test_func):
    optimizer_init_test_func(NuclearReactionOptimization)

def test_nro_nfi_phase(default_optimizer, position, fitness):
    optimizer = default_optimizer(NuclearReactionOptimization)
    optimizer.current_epoch = 0
    optimizer.g_best_pos = np.array([-0.99446959, -0.90140898])
    result = optimizer.nfi_phase(position, fitness)
    assert len(result) == 10
    assert result[0].shape == (2,)

def test_nro_nfu_phase(default_optimizer, position, fitness, g_best_pos):
    optimizer = default_optimizer(NuclearReactionOptimization)
    optimizer.current_epoch = 1
    optimizer.g_best_pos = g_best_pos
    result = optimizer.nfu_phase(position, fitness)
    assert len(result) == 10
    assert result[0].shape == (2,)

def test_nro_calculate_pc(default_optimizer, position, fitness, g_best_pos):
    optimizer = default_optimizer(NuclearReactionOptimization)
    optimizer.current_epoch = 2
    optimizer.g_best_pos = g_best_pos
    result = optimizer.calculate_pc(position, fitness)
    assert len(result) == 10
    assert result[0].shape == (2,)

def test_nro_ask(optimizer_ask_test_func):
    optimizer_ask_test_func(NuclearReactionOptimization)

def test_nro_train(optimizer_optimize_test_func):
    optimizer_optimize_test_func(NuclearReactionOptimization)

max_stars_repo_name: pfnet-research/batch-metaheuristics

max_issues_repo_name: pfnet-research/batch-metaheuristics

max_forks_repo_name: pfnet-research/batch-metaheuristics


text:
import numpy as np
import cv2
import scipy.spatial

def get_color_for_point(point_coords, list_of_point_centers, list_of_colors):
    """
    get_color_for_point() computes an RGB color value for a point in the
    (-1, 1) 2D plane, based on a set of RGB color values, defined on particular
    points of the same 2D plane.
    :param point_coords: coordinates for the point for which we want to
    calculate its color
    :param list_of_point_centers: list of point coodinates
    [[x1, y1], ..., [xN, yMN] of the of the aforementioned colors
    :param list_of_colors:  list of RGB color values [[R1, G1, B1], ...,
    [RN, GN, BN]] for the N points in the 2D plane (see prev atribute)
    :return: interpolated RGB color for the input point (1st arg)
    """
    color = np.array([0.0, 0.0, 0.0])
    # get distances of the "query" point from all other points
    distances = scipy.spatial.distance.cdist([point_coords],
                                             list_of_point_centers)[0]

    # get weights and compute new RGB value as weighted sum:
    weights = 1 / (distances + 0.1)
    for ic, c in enumerate(list_of_colors):
        color += (np.array(c) * weights[ic])
    color /= (np.sum(weights))
    sum_color = np.sum(color)
    required_sum_color = 600.0
    if color.max() * (required_sum_color/sum_color) <= 255:
        color *= (required_sum_color/sum_color)
    else:
        color *= (255/(color.max()))
    return color

def create_2d_color_map(list_of_points, list_of_colors, height, width):
    """
    create_2d_color_map() creates a colormap by interpolating RGB color values,
    given a list of colors to be defined on particular points of the 2D
    plane.
    :param list_of_points: list of point coodinates
    [[x1, y1], ..., [xN, yMN] of the of the aforementioned colors
    :param list_of_colors: list of RGB color values [[R1, G1, B1], ...,
    [RN, GN, BN]] for the N points in the 2D plane (see prev atribute)
    :param height: output image height
    :param width:  output image weight
    :return: estimated color image
    """
    rgb = np.zeros((height, width, 3)).astype("uint8")
    c_x = int(width / 2)
    c_y = int(height / 2)
    step = 5
    win_size = int((step-1) / 2)
    for i in range(len(list_of_points)):
        rgb[c_y - int(list_of_points[i][1] * height / 2),
            c_x + int(list_of_points[i][0] * width / 2)] = list_of_colors[i]
    for y in range(win_size, height - win_size, step):
        for x in range(win_size, width - win_size, step):
            x_real = (x - width / 2) / (width / 2)
            y_real = (height / 2 - y ) / (height / 2)
            color = get_color_for_point([x_real, y_real], list_of_points,
                                        list_of_colors)
            rgb[y - win_size - 1 : y + win_size + 1,
                x - win_size - 1 : x + win_size + 1] = color
    bgr = cv2.cvtColor(rgb, cv2.COLOR_BGR2RGB)
    return bgr

if __name__ == "__main__":
    colors = {"coral": [255,127,80],
              "pink": [255, 192, 203],
              "orange": [255, 165, 0],
              "blue": [0, 0, 205],
              "green": [0, 205, 0],
              "red": [205, 0, 0],
              "yellow": [204, 204, 0]}
    angry_pos = [-0.8, 0.5]
    fear_pos = [-0.3, 0.8]
    happy_pos = [0.6, 0.6]
    calm_pos = [0.4, -0.5]
    sad_pos = [-0.6, -0.4]
    bgr = create_2d_color_map([angry_pos, fear_pos, happy_pos,
                               calm_pos, sad_pos],
                              [colors["red"],  colors["yellow"],
                               colors["orange"], colors["green"],
                               colors["blue"]], 200, 200)
    cv2.imshow('Signal', bgr)
    ch = cv2.waitKey(10000)