text:
import unittest
import numpy as np
import pandas as pd
from pyrolite.util.text import *

class TestRemovePrefix(unittest.TestCase):
    def test_prefix_present(self):
        pass

    def test_prefix_notpresent(self):
        #
        # for prefix in ['A_A_', 'B_', 'C_']:
        #    with

        # self.assertFalse(s.startswith(prefix))
        pass

    def test_double_prefix(self):
        """Should just remove one prefix."""

        pass

class TestNormaliseWhitespace(unittest.TestCase):
    def test_whitepace_removal(self):
        pass

    def test_whitespace_preservation(self):
        pass

class TestToWidth(unittest.TestCase):
    def test_width_spec(self):
        """
        Check that the output width is as specifed.
        For strings including whitespace, it should
        preserve word structure.
        """

        s = "*- " * 100
        for width in [1, 10, 79]:
            with self.subTest(width=width):
                out = to_width(s, width=width)
                w = len(out.splitlines()[0])
                self.assertTrue(w <= width)
                self.assertTrue(w >= width - len(s))

class TestQuotedString(unittest.TestCase):
    """Tests for quoted string utility."""

    def test_quoted_string(self):
        """Check that quoted strings operate correctly."""

        for string in [
            "singlephrase",
            "double phrase",
            "tri ple phrase",
            "singlephrase",
            "double phrase",
            "tri ple phrase",
            """singlephrase""",
            """double phrase""",
            """tri ple phrase""",
        ]:

            with self.subTest(string=string):

                quotes = ["'", '"']
                stringcontent = "".join(
                    i for i in quoted_string(string) if i not in quotes
                )
                # Check same content
                self.assertEqual(stringcontent, "".join(i for i in string))

                contains = [[s in q for s in quoted_string(string)] for q in quotes]
                # Check there's at least two of the same quotes.
                self.assertTrue((np.array(contains).sum() >= 2).any())

class TestTitlecase(unittest.TestCase):
    """Tests for titlecase string utility."""

    def test_single_word(self):
        """Check single word cases operate correctly."""
        for string in ["lowercase", "UPPERCASE", "CamelCase", "Titlecase"]:
            with self.subTest(string=string):
                # String content doesnt change
                self.assertEqual(titlecase(string).lower(), string.lower())
                # Single word capitalisation
                self.assertEqual(
                    titlecase(string), string[0].upper() + string[1:].lower()
                )

    def test_multi_word(self):
        """Check multiword case operates correctly."""
        for string in ["lower case", "UPPER CASE", "Camel Case", "Title case"]:
            with self.subTest(string=string):
                # String content doesnt change
                self.assertEqual(
                    titlecase(string).lower(), string.lower().replace(" ", "")
                )

                expected = "".join(
                    [st[0].upper() + st[1:].lower() for st in string.split(" ")]
                )
                self.assertEqual(titlecase(string), expected)

    def test_valid_split_delimiters(self):
        """Check split delimiters operate correctly."""
        for string in [
            "lower case",
            "lower-case",
            "lower_case",
            "lower\tcase",
            "lower\ncase",
        ]:
            with self.subTest(string=string):
                self.assertEqual(titlecase(string).lower(), "lowercase")

    @unittest.expectedFailure
    def test_invalid_split_delimiters(self):
        """Check split delimiters operate correctly."""
        for string in ["lower/case", "lower\case", "lower&case"]:
            with self.subTest(string=string):
                self.assertEqual(titlecase(string).lower(), "lowercase")

    def test_initial_space(self):
        """Check that strings are stripped effectively."""
        pass

    def test_join_characters(self):
        """Check join charaters operate correctly."""
        pass

    def test_capitalize_first_word(self):
        """Check capitalize_first operates correctly."""
        pass

    def test_exceptions(self):
        """Check execptions operate correctly."""
        pass

    def test_abbreviations(self):
        """Check abbreviations operate correctly."""
        pass

    def test_original_camelcase(self):
        """Check whether original camelcase is preserved."""

class TestParseEntry(unittest.TestCase):
    """Tests the regex parser for data munging string --> value conversion."""

    def setUp(self):
        self.regex = r"(\s)*?(?P<value>[\w]+)(\s)*?"

    def test_single_entry(self):
        for value in ["A", " A ", " A  ", " A _1", "A .[1]"]:
            with self.subTest(value=value):
                parsed = parse_entry(value, regex=self.regex)
                self.assertEqual(parsed, "A")

    def test_multiple_value_return(self):
        delimiter = ","
        for value in ["A, B", " A ,B ", " A , B", " A ,B _", "A, B.[1]"]:
            with self.subTest(value=value):
                parsed = parse_entry(
                    value, regex=self.regex, delimiter=",", first_only=False
                )
                self.assertEqual(parsed, ["A", "B"])

    def test_first_only(self):
        delimiter = ","
        for value, fo, expect in [("A, B", True, "A"), ("A, B", False, ["A", "B"])]:
            with self.subTest(value=value, fo=fo, expect=expect):
                parsed = parse_entry(
                    value, regex=self.regex, delimiter=",", first_only=fo
                )
                self.assertEqual(parsed, expect)

    def test_delimiters(self):
        for value, delim in [("A, B", ","), ("A; B", ";"), ("A- B", "-"), ("A B", " ")]:
            with self.subTest(value=value, delim=delim):
                parsed = parse_entry(
                    value, regex=self.regex, delimiter=delim, first_only=False
                )
                self.assertEqual(parsed, ["A", "B"])

    def test_replace_nan(self):
        for null_value in [np.nan, None]:
            for n in [np.nan, "None", None]:
                with self.subTest(n=n, null_value=null_value):
                    parsed = parse_entry(
                        null_value, regex=self.regex, delimiter=",", replace_nan=n
                    )
                    if n is None:
                        self.assertTrue(parsed is None)
                    elif isinstance(n, str):
                        self.assertTrue(parsed == n)
                    else:
                        self.assertTrue(np.isnan(parsed))

    def test_values_only(self):
        regex = r"(\s)*?(?P<value>[\w]+)(\s)*?"
        for value, vo, expect in [
            ("A, B", True, ["A", "B"]),
            ("A, B", False, [{"value": "A"}, {"value": "B"}]),
        ]:
            with self.subTest(value=value):
                parsed = parse_entry(
                    value, regex=regex, delimiter=",", first_only=False, values_only=vo
                )
                self.assertEqual(parsed, expect)

class TestStringVariations(unittest.TestCase):
    def setUp(self):
        self.single = "single"
        self.multiple = ["Multiple-a", "multiple-b"]

    def test_single(self):
        self.assertEqual(
            string_variations(self.single), string_variations([self.single])
        )

    def test_preprocess(self):
        for ps in [["lower"], ["upper"], ["upper", "lower"]]:
            string_variations(self.single, preprocess=ps)

    def test_multiple(self):
        string_variations(self.multiple)

class TestSplitRecords(unittest.TestCase):
    """Tests the regex parser for poorly formatted records."""

    def setUp(self):
        pass

    def test_single_entry(self):
        pass

    def test_delimiters(self):
        pass

if __name__ == "__main__":
    unittest.main()

max_stars_repo_name: bomtuckle/pyrolite

max_issues_repo_name: bomtuckle/pyrolite

max_forks_repo_name: bomtuckle/pyrolite


text:
import unittest
import numpy as np
from skywinder.communication import packet_classes

class test_sip_packet_malfunctions(unittest.TestCase):
    def empty_packet_test(self):
        buffer = '\xfa\xff\x01\x00\x00\x00\x01'
        packets, remainder = packet_classes.get_packets_from_buffer(buffer,
                                                                    packet_class=packet_classes.GSEPacket,
                                                                    start_byte=packet_classes.GSEPacket.START_BYTE)
        assert (remainder == '')
        assert (packets[0].to_buffer() == buffer)

    def sip_packet_in_data_test(self):
        initial_buffer = '\xfa\xff\x01\x00\x00\x00\x01'
        np.random.seed(0)
        random_msg = np.random.randint(0, 255, size=10000).astype('uint8').tostring()
        buffer = random_msg[:(len(random_msg) / 2)] + initial_buffer + random_msg[(len(random_msg) / 2):]
        packets, remainder = packet_classes.get_packets_from_buffer(buffer,
                                                                    packet_class=packet_classes.GSEPacket,
                                                                    start_byte=packet_classes.GSEPacket.START_BYTE)

        assert (packets[0].to_buffer() == initial_buffer)

    def false_start_test(self):
        buffer = '\xfa\xfa\xff\x01\x00\x00\x00\x01'
        packets, remainder = packet_classes.get_packets_from_buffer(buffer,
                                                                    packet_class=packet_classes.GSEPacket,
                                                                    start_byte=packet_classes.GSEPacket.START_BYTE)
        packet = packets[0]
        print('%r' % packet.to_buffer())
        print(packet.checksum)
        print('%r' %packet.payload)
        print(packet.payload_length)
        assert (packets[0].to_buffer() == '\xfa\xff\x01\x00\x00\x00\x01')

        # Interesting corner case - this is a valid packet... except for the unused byte and origin.

    def insufficient_length_for_length_byte_test(self):
        buffer = '\xfa\xff\x01'
        packets, remainder = packet_classes.get_packets_from_buffer(buffer,
                                                                    packet_class=packet_classes.GSEPacket,
                                                                    start_byte=packet_classes.GSEPacket.START_BYTE)
        assert (remainder == buffer)
        assert (packets == [])

    def insufficient_length_for_length_byte_test_2(self):
        buffer = '\xfa\xff\x01\x00\x00\x03\x00'
        packets, remainder = packet_classes.get_packets_from_buffer(buffer,
                                                                    packet_class=packet_classes.GSEPacket,
                                                                    start_byte=packet_classes.GSEPacket.START_BYTE)
        assert (remainder == buffer)
        assert (packets == [])

    def insufficient_length_for_checksum_test(self):
        buffer = '\xfa\xff\x01\x00\x00\x00'
        packets, remainder = packet_classes.get_packets_from_buffer(buffer,
                                                                    packet_class=packet_classes.GSEPacket,
                                                                    start_byte=packet_classes.GSEPacket.START_BYTE)
        assert (remainder == buffer)
        assert (packets == [])

    def bad_id_byte_test(self):
        buffer = '\xfa\x99\x01\x00\x00\x00\x01'
        packets, remainder = packet_classes.get_packets_from_buffer(buffer,
                                                                    packet_class=packet_classes.GSEPacket,
                                                                    start_byte=packet_classes.GSEPacket.START_BYTE)
        assert (remainder == '\x99\x01\x00\x00\x00\x01')
        assert (packets == [])

    def bad_checksum_test(self):
        buffer = '\xfa\xff\x01\x00\x00\x00\x07'
        packets, remainder = packet_classes.get_packets_from_buffer(buffer,
                                                                    packet_class=packet_classes.GSEPacket,
                                                                    start_byte=packet_classes.GSEPacket.START_BYTE)
        assert (remainder == '\xff\x01\x00\x00\x00\x07')
        assert (packets == [])

    def long_sip_packet_test(self):
        sync = [0xFA, 0xFF]
        header = [0x01, 0x00]
        data = list(range(0, 100))
        length = [0x00, len(data)]  # assumes payload is < 255 bytes
        checksum_data = np.array(header + length + data, dtype='uint8')
        checksum = int(np.sum(checksum_data, dtype='uint8'))
        #print "total sum:", np.sum(checksum_data)
        assert (checksum < 256)
        buffer = np.array(sync + header + length + data + [checksum], dtype='uint8').tostring()
        packets, remainder = packet_classes.get_packets_from_buffer(buffer,
                                                                    packet_class=packet_classes.GSEPacket,
                                                                    start_byte=packet_classes.GSEPacket.START_BYTE)
        assert (remainder == '')
        assert (packets[0].to_buffer() == buffer)

class TestFilePackets(unittest.TestCase):
    def setUp(self):
        np.random.seed(0)
        self.random_msg = np.random.randint(0, 255, size=10000).astype('uint8')
        self.random_msg = self.random_msg.tostring()
        self.random_msg = '\x02' + self.random_msg[1:]

    def run_through_good_test(self):
        # Tests filepacket
        packets = []
        packet_size = 1000
        num_packets = int(np.ceil(len(self.random_msg) / packet_size))
        for i in range(num_packets):
            msg = self.random_msg[(i * packet_size):((i + 1) * packet_size)]
            packet = packet_classes.FilePacket(file_id=0, packet_number=i,
                                               total_packet_number=num_packets, payload=msg)
            packets.append(packet)

        buffer = ''
        for packet in packets:
            buffer += packet.to_buffer()
        packets_after, remainder = packet_classes.get_packets_from_buffer(buffer, packet_class=packet_classes.FilePacket,
                                                start_byte=packet_classes.FilePacket._valid_start_byte)
        data_buffer = ''.join([packet.payload for packet in packets_after])
        assert (data_buffer == self.random_msg)

        # g = gse_receiver.GSEReceiver()
        # file_after = g.assemble_file_from_packets(packets_after)
        # assert (np.array_equal(self.random_msg, file_after.payload) == True)

    # def two_files_shuffled_together_test(self):
    #     # Tests two filepackets shuffled together
    #     encoded_data = cobs_encoding.encode_data(self.random_msg, escape_character=constants.SYNC_BYTE)
    #     chunks0 = hirate_sending_methods.chunk_data_by_size(chunk_size=1000, start_byte=constants.SYNC_BYTE, file_id=0,
    #                                                         data=encoded_data)
    #     chunks1 = hirate_sending_methods.chunk_data_by_size(chunk_size=1000, start_byte=constants.SYNC_BYTE, file_id=1,
    #                                                         data=encoded_data)
    #     chunks = chunks0 + chunks1
    #     random.seed(0)
    #     random.shuffle(chunks)
    #     buffer = ''
    #     for chunk in chunks:
    #         buffer += chunk
    #     packets = hirate_receiving_methods.get_packets_from_buffer(buffer, start_byte=chr(constants.SYNC_BYTE))
    #     files = hirate_receiving_methods.packets_to_files(packets)
    #
    #     decoded_data0 = cobs_encoding.decode_data(files[0], escape_character=constants.SYNC_BYTE)
    #     unpacked_data0 = struct.unpack(('%dB' % len(decoded_data0)), decoded_data0)
    #     unpacked_data_array0 = np.array(unpacked_data0).astype('uint8')
    #     assert (np.array_equal(self.random_msg, unpacked_data_array0) == True)
    #
    #     decoded_data1 = cobs_encoding.decode_data(files[1], escape_character=constants.SYNC_BYTE)
    #     unpacked_data1 = struct.unpack(('%dB' % len(decoded_data1)), decoded_data1)
    #     unpacked_data_array1 = np.array(unpacked_data1).astype('uint8')
    #     assert (np.array_equal(self.random_msg, unpacked_data_array1) == True)

ext: py

max_stars_repo_name: PolarMesosphericClouds/SkyWinder

max_issues_repo_name: PolarMesosphericClouds/SkyWinder

max_forks_repo_name: PolarMesosphericClouds/SkyWinder

reason: import numpy


text:
"""
Processes for Species distribution
Author: Nils Hempelmann ( info@nilshempelmann.de )
"""
from pywps import Process
from pywps import LiteralInput
from pywps import ComplexInput, ComplexOutput
from pywps import Format
from pywps.inout.literaltypes import AllowedValue
from pywps.app.Common import Metadata

from flyingpigeon import sdm
from flyingpigeon.sdm import _SDMINDICES_
from flyingpigeon.utils import rename_complexinputs
from flyingpigeon.utils import archive, archiveextract, download
from flyingpigeon.visualisation import map_gbifoccurrences
from flyingpigeon.visualisation import map_PAmask
from flyingpigeon.visualisation import pdfmerge, concat_images

from flyingpigeon.log import init_process_logger

import tempfile
import logging
LOGGER = logging.getLogger("PYWPS")

class SDMcsvProcess(Process):
    def __init__(self):
        inputs = [
            ComplexInput('resource', 'Resource',
                         abstract='NetCDF Files or archive (tar/zip) containing netCDF files.',
                         metadata=[Metadata('Info')],
                         min_occurs=1,
                         max_occurs=1000,
                         supported_formats=[
                             Format('application/x-netcdf'),
                             Format('application/x-tar'),
                             Format('application/zip'),
                         ]),

            LiteralInput("gbif", "GBIF csv file",
                         abstract="GBIF table (csv) with tree occurence \
                         (output of 'GBIF data fetch' process )",
                         data_type='string',
                         min_occurs=1,
                         max_occurs=1,
                        #  default='http://localhost:8090/wpsoutputs/flyingpigeon/output_csv-abe15f64-c30d-11e6-bf63-142d277ef1f3.csv'
                         ),

            LiteralInput("indices", "Indices",
                         abstract="Climate indices related to growth conditions \
                                    of tree species",
                         default='TG_JJA',
                         data_type='string',
                         min_occurs=1,
                         max_occurs=10,
                         allowed_values=_SDMINDICES_
                         ),

            LiteralInput("period", "Reference period",
                         abstract="Reference period for climate conditions\
                         (all = entire timeseries)",
                         default="all",
                         data_type='string',
                         min_occurs=1,
                         max_occurs=1,
                         allowed_values=['all', '1951-1980', '1961-1990',
                                         '1971-2000', '1981-2010']
                         ),

            LiteralInput("archive_format", "Archive format",
                         abstract="Result files will be compressed into archives.\
                                  Choose an appropriate format",
                         default="tar",
                         data_type='string',
                         min_occurs=1,
                         max_occurs=1,
                         allowed_values=['zip', 'tar']
                         )
        ]
        outputs = [

            ComplexOutput("output_gbif", "Graphic of GBIF coordinates",
                          abstract="PNG graphic file showing the presence of tree species\
                                    according to the CSV file",
                          supported_formats=[Format('image/png')],
                          as_reference=True,
                          ),

            ComplexOutput("output_PA", "Graphic of PA mask",
                          abstract="PNG graphic file showing PA mask generated based on\
                                    netCDF spatial increment",
                          supported_formats=[Format('image/png')],
                          as_reference=True,
                          ),

            ComplexOutput("output_indices", "Climate indices for growth conditions over all timesteps",
                          abstract="Archive (tar/zip) containing calculated climate indices",
                          supported_formats=[Format('application/x-tar'),
                                             Format('application/zip')
                                             ],
                          as_reference=True,
                          ),

            ComplexOutput("output_reference", "Climate indices for growth conditions of reference period",
                          abstract="Archive (tar/zip) containing calculated climate indices",
                          supported_formats=[Format('application/x-tar'),
                                             Format('application/zip')
                                             ],
                          as_reference=True,
                          ),

            ComplexOutput("output_prediction", "predicted growth conditions",
                          abstract="Archive containing files of the predicted\
                                     growth conditions",
                          supported_formats=[Format('application/x-tar'),
                                             Format('application/zip')
                                             ],
                          as_reference=True,
                          ),

            ComplexOutput("output_info", "GAM statistics information",
                          abstract="Graphics and information of the learning statistics",
                          supported_formats=[Format("application/pdf")],
                          as_reference=True,
                          ),
            ComplexOutput('output_log', 'Logging information',
                          abstract="Collected logs during process run.",
                          as_reference=True,
                          supported_formats=[Format('text/plain')]
                          )
        ]

        super(SDMcsvProcess, self).__init__(
            self._handler,
            identifier="sdm_csv",
            title="Species distribution Model (GBIF-CSV table as input)",
            version="0.10",
            metadata=[
                Metadata("LWF", "http://www.lwf.bayern.de/"),
                Metadata(
                    "Doc",
                    "http://flyingpigeon.readthedocs.io/en/latest/descriptions/index.html#species-distribution-model"),
                Metadata("paper",
                         "http://www.hindawi.com/journals/jcli/2013/787250/"),
                Metadata("Tutorial",
                         "http://flyingpigeon.readthedocs.io/en/latest/tutorials/sdm.html"),
            ],
            abstract="Indices preparation for SDM process",
            inputs=inputs,
            outputs=outputs,
            status_supported=True,
            store_supported=True,
            )

    def _handler(self, request, response):
        init_process_logger('log.txt')
        response.outputs['output_log'].file = 'log.txt'
        response.update_status('Start process', 0)

        try:
            LOGGER.info('reading the arguments')
            resources = archiveextract(
                resource=rename_complexinputs(request.inputs['resource']))
            period = request.inputs['period']
            period = period[0].data
            indices = [inpt.data for inpt in request.inputs['indices']]
            archive_format = request.inputs['archive_format'][0].data
            LOGGER.info("all arguments read in nr of files in resources: %s" % len(resources))
        except Exception:
            LOGGER.exception('failed to read in the arguments')
            raise

        try:
            gbif_url = request.inputs['gbif'][0].data
            csv_file = download(gbif_url)
            LOGGER.info('CSV file fetched sucessfully: %s' % csv_file)
        except:
            LOGGER.exception('failed to fetch GBIF file')

        try:
            response.update_status('read in latlon coordinates', 10)
            latlon = sdm.latlon_gbifcsv(csv_file)
            LOGGER.info('got occurence coordinates %s ' % csv_file)
        except:
            LOGGER.exception('failed to extract the latlon points from file: %s' % (csv_file))

        try:
            response.update_status('plot map', 20)
            occurence_map = map_gbifoccurrences(latlon)
            LOGGER.info('GBIF occourence ploted')
        except:
            LOGGER.exception('failed to plot occurence map')

        #################################
        # calculate the climate indices
        #################################

        # get the indices
        try:
            response.update_status('start calculation of indices', 30)
            ncs_indices = sdm.get_indices(resource=resources, indices=indices)
            LOGGER.info('indice calculation done')
        except:
            msg = 'failed to calculate indices'
            LOGGER.exception(msg)
            raise Exception(msg)

        try:
            # sort indices
            indices_dic = sdm.sort_indices(ncs_indices)
            LOGGER.info('indice files sorted in dictionary')
        except:
            msg = 'failed to sort indices'
            LOGGER.exception(msg)
            indices_dic = {'dummy': []}

        ncs_references = []
        species_files = []
        stat_infos = []
        PAmask_pngs = []

        response.update_status('Start processing for %s Datasets' % len(indices_dic.keys()))
        for count, key in enumerate(indices_dic.keys()):
            try:
                staus_nr = 40 + count*10
                response.update_status('Start processing of %s' % key, staus_nr)
                ncs = indices_dic[key]
                LOGGER.info('with %s files' % len(ncs))

                try:
                    response.update_status('generating the PA mask', 20)
                    PAmask = sdm.get_PAmask(coordinates=latlon, nc=ncs[0])
                    LOGGER.info('PA mask sucessfully generated')
                except:
                    LOGGER.exception('failed to generate the PA mask')

                try:
                    response.update_status('Ploting PA mask', 25)
                    PAmask_pngs.extend([map_PAmask(PAmask)])
                except:
                    LOGGER.exception('failed to plot the PA mask')

                try:
                    ncs_reference = sdm.get_reference(ncs_indices=ncs, period=period)
                    ncs_references.extend(ncs_reference)
                    LOGGER.info('reference indice calculated %s '
                                % ncs_references)
                except:
                    msg = 'failed to calculate the reference'
                    LOGGER.exception(msg)
                    # raise Exception(msg)

                try:
                    gam_model, predict_gam, gam_info = sdm.get_gam(ncs_reference, PAmask)
                    stat_infos.append(gam_info)
                    response.update_status('GAM sucessfully trained', staus_nr + 5)
                except:
                    msg = 'failed to train GAM for %s' % (key)
                    LOGGER.debug(msg)

                try:
                    prediction = sdm.get_prediction(gam_model, ncs)
                    response.update_status('prediction done', staus_nr + 7)
                except:
                    msg = 'failed to predict tree occurence'
                    LOGGER.exception(msg)
                    # raise Exception(msg)
                #
                # try:
                #     response.update_status('land sea mask for predicted data',  staus_nr + 8)
                #     from numpy import invert, isnan, nan, broadcast_arrays  # , array, zeros, linspace, meshgrid
                #     mask = invert(isnan(PAmask))
                #     mask = broadcast_arrays(prediction, mask)[1]
                #     prediction[mask is False] = nan
                # except:
                #     LOGGER.debug('failed to mask predicted data')

                try:
                    species_files.append(sdm.write_to_file(ncs[0], prediction))
                    LOGGER.info('Favourabillity written to file')
                except:
                    msg = 'failed to write species file'
                    LOGGER.debug(msg)
                    raise Exception(msg)

            except:
                msg = 'failed to process SDM chain for %s ' % key
                LOGGER.exception(msg)

        try:
            archive_indices = archive(ncs_indices, format=archive_format)
            LOGGER.info('indices added to archive')
        except:
            msg = 'failed adding indices to archive'
            LOGGER.exception(msg)
            archive_indices = tempfile.mkstemp(suffix='.tar', prefix='foobar-', dir='.')

        try:
            archive_references = archive(ncs_references, format=archive_format)
            LOGGER.info('indices reference added to archive')
        except:
            msg = 'failed adding reference indices to archive'
            LOGGER.exception(msg)
            archive_references = tempfile.mkstemp(suffix='.tar', prefix='foobar-', dir='.')

        try:
            archive_prediction = archive(species_files, format=archive_format)
            LOGGER.info('species_files added to archive')
        except:
            msg = 'failed adding species_files indices to archive'
            LOGGER.exception(msg)
            raise Exception(msg)

        try:
            stat_infosconcat = pdfmerge(stat_infos)
            LOGGER.debug('pngs %s' % PAmask_pngs)
            PAmask_png = concat_images(PAmask_pngs, orientation='h')
            LOGGER.info('stat infos pdfs and mask pngs merged')
        except:
            LOGGER.exception('failed to concat images')
            _, stat_infosconcat = tempfile.mkstemp(suffix='.pdf', prefix='foobar-', dir='.')
            _, PAmask_png = tempfile.mkstemp(suffix='.png', prefix='foobar-', dir='.')

        # self.output_csv.setValue(csv_file)
        response.outputs['output_gbif'].file = occurence_map
        response.outputs['output_PA'].file = PAmask_png
        response.outputs['output_indices'].file = archive_indices
        response.outputs['output_reference'].file = archive_references
        response.outputs['output_prediction'].file = archive_prediction
        response.outputs['output_info'].file = stat_infosconcat

        response.update_status('done', 100)
        return response

max_stars_repo_name: Zeitsperre/flyingpigeon

max_issues_repo_name: Zeitsperre/flyingpigeon

max_forks_repo_name: Zeitsperre/flyingpigeon

reason: from numpy


text:
import numpy as np
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit
import time

class RandomWalkWithAbsorbingBarrier:
    def __init__(self, length):
        self.length = length
        self.x_initial = 0
        self.life_time = 0

    def render(self, x_0):
        self.x_initial = x_0
        self.life_time = 0
        if not(0 < x_0 < (self.length - 1)):
            return

        random_mean_size = 2 * x_0 * (self.length - 1 - x_0)
        position = x_0
        while True:
            steps = np.random.choice([1, -1], random_mean_size)
            for step in steps:
                position += step
                self.life_time += 1
                if not(0 < position < (self.length - 1)):
                    return

def model_parabolic_func(t, a, b, c):
    return a * ((t - b) ** 2) + c

def calc_mean_life_time(length=20, step=1, sample_numbers=100000, file_name=None):
    if file_name is None:
        x_s = np.arange(0, length, step, dtype=int)
        model = RandomWalkWithAbsorbingBarrier(length)
        sample_life_times_ensemble = np.zeros((len(x_s), sample_numbers), dtype=np.int)

        for (x_i, x) in enumerate(x_s):
            print(str(x) + ':')
            for sample_number in range(sample_numbers):
                print('\r\t' + str(sample_number), end="")
                model.render(x)
                sample_life_times_ensemble[x_i][sample_number] = model.life_time

            print(end='\n')

        data = {
            'x_s': x_s,
            'length': length,
            'step': step,
            'life_times': sample_life_times_ensemble,
            'numbers': sample_numbers
        }
        np.save("data/q3_" + str(length) + '_' + str(step) + "_" + str(sample_numbers) + "_ensemble", data)
    else:
        data = np.load('data/q3_' + file_name + '_ensemble.npy', allow_pickle=True).tolist()
        x_s = data['x_s']
        length = data['length']
        step = data['step']
        sample_life_times_ensemble = data['life_times']
        sample_numbers = data['numbers']

    life_time_avg = np.average(sample_life_times_ensemble, axis=1)

    life_time_fit_para, life_time_fit_error = curve_fit(model_parabolic_func, x_s, life_time_avg)
    life_time_fit_error = np.diag(life_time_fit_error)
    life_time_fit = life_time_fit_para[0] * ((x_s - life_time_fit_para[1]) ** 2) + life_time_fit_para[2]

    plt.plot(x_s, life_time_fit, linestyle='--', color='g')
    plt.plot(x_s, life_time_avg, linestyle='', marker='.', markersize=5)
    plt.xlabel(r'$x_0$')
    plt.ylabel(r'<Life Time>')
    plt.legend(['Fitted line', 'Samples'])
    plt.savefig(
        "images/q3_life_time_" + str(length) + '_' + str(step) + "_" + str(sample_numbers) + '.png')
    plt.show()

    print('<Life_Time(x_0)> = ' + str(life_time_fit_para[0]) + ' * (x_0 - ' + str(life_time_fit_para[1]) + ') ^ 2 + '+
          str(life_time_fit_para[2]))

start = time.time()
calc_mean_life_time(20, 1, 100000)
end = time.time()

print('Time: ' + str(end - start))

reason: import numpy,from scipy


text:
"""
/irahorecka/api/craigslisthousing/update/score.py
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Module to provide scores to posts in the Craigslist housing database table.
"""

import numpy as np
from sqlalchemy.sql import func, and_

from irahorecka.models import db, CraigslistHousing

def write_craigslist_housing_score(site, areas):
    """ENTRY POINT: Assigns and writes Craigslist housing value scores to posts. Posts
    without a score are set to 0."""
    # Build base query after applying preliminary filters.
    query = preliminary_filter(CraigslistHousing, CraigslistHousing.query)
    query_site = query.filter(CraigslistHousing.site == site)
    query_site_ft2 = query_site.filter(CraigslistHousing.ft2 != 0)
    query_site_sans_ft2 = query_site.filter(CraigslistHousing.ft2 == 0)

    for area in areas:
        query_area = query_site.filter(CraigslistHousing.area == area)
        query_area_ft2 = query_site_ft2.filter(CraigslistHousing.area == area)
        query_area_sans_ft2 = query_site_sans_ft2.filter(CraigslistHousing.area == area)

        # Calculate score for posts with price and ft2.
        with Ft2(CraigslistHousing, query_site_ft2, query_area_ft2) as ft2:
            ft2.write_score(query_area_ft2)
        # Posts with and without filters are scored differently - normalize scores before grouping.
        normalize_score(CraigslistHousing, query_area_ft2, -100, 100)
        # Calculate score for posts with price without ft2.
        with Bedrooms(CraigslistHousing, query_site, query_area) as bedrooms:
            bedrooms.write_score(query_area_sans_ft2)
        # Posts without ft2 have scores not as influential as posts with ft2 - set range to -90, 90.
        normalize_score(CraigslistHousing, query_area_sans_ft2, -90, 90)

    # Set null scores to 0 to maintain constant numeric datatype.
    query_site.filter(CraigslistHousing.score.is_(None)).update({CraigslistHousing.score: 0})
    db.session.commit()

def preliminary_filter(model, query):
    """Performs preliminary filters to a query and model. Use to clean up query prior to performing
    statistical analysis."""
    # If 'studio' is anywhere in the title, force bedrooms to reflect studio property.
    query.filter(func.lower(model.title).contains("studio")).update({model.bedrooms: 0}, synchronize_session="fetch")
    return query

class Score:
    """Base class for calculating score of a post."""

    def __init__(self, model, query_site, query_area):
        # `query_area` must be a child query of `query_site`.
        self.model = model
        self.query_site = query_site
        self.query_area = query_area

    def __enter__(self):
        """Sets up instance to have queries work with 0.5 bedrooms instead of 0. This is because of
        the scoring criteria in `_calculate_post_score`, where we work with the logarithmic value of
        number of bedrooms."""
        self.query_site.filter(self.model.bedrooms == 0).update({self.model.bedrooms: 0.5})
        self.query_area.filter(self.model.bedrooms == 0).update({self.model.bedrooms: 0.5})
        return self

    def __exit__(self, exc_type, exc_value, exc_tb):
        """Sets queries with 0.5 bedrooms (from `__enter__`) to 0."""
        self.query_site.filter(self.model.bedrooms == 0.5).update({self.model.bedrooms: 0})
        self.query_area.filter(self.model.bedrooms == 0.5).update({self.model.bedrooms: 0})

    def _calculate_post_score(self, site_z_score, area_z_score):
        """Calculates value score for every post. Read description below for calc breakdown:
        - Gets Z-Scores for posts within a site and area.
            - Area Z-Score is multiplied with 0.25 coefficient, Site value with 0.65 coefficient.
        - Score equation:
            = 0.25 * (1 - (0.25 * z_score_within_area)) + 0.65 * (1 - (0.65 * z_score_within_site)) + 0.1 * (1 + math.log(num_bedrooms)) - 1
        - Purely average post should equate to a score of 1.0, meaning completely average price within site, area, and has 1 bedroom.
        - Score more than 0 is GOOD, less than 0 is BAD."""
        site_weight = 0.25 * (1 - (0.25 * site_z_score))
        area_weight = 0.65 * (1 - (0.65 * area_z_score))
        bedroom_weight = 0.1 * (1 + func.log(self.model.bedrooms))
        # Subtract 1, which is the neutral score.
        return site_weight + area_weight + bedroom_weight - 1

    @staticmethod
    def _get_output_within_percentile(fn, *args, perc_min=5, perc_max=95):
        """Takes lower and upper percentile limit and returns a tuple of lists where the output
        (i.e. fn(arg1, arg2))) is within the specified percentile range."""
        output = fn(*args)
        low = np.percentile(output, perc_min)
        high = np.percentile(output, perc_max)
        desired_percentile = np.where(np.logical_and(output >= low, output <= high))
        return output[desired_percentile]

class Ft2(Score):
    """Calculates score for posts with ft2 attribute. Inherits from `Score`."""

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def write_score(self, query_write):
        """Writes score to posts within `query_write`."""
        # Filter posts to allow for logarithmic operations.
        query_write = self._filter_query_for_log_calc(query_write)
        summary = self._get_log_postvalue_summary()
        log_postvalue = func.log(
            self._postvalue_fn(func.log, func.sqrt, self.model.price, self.model.ft2, self.model.bedrooms)
        )
        site_z_score = (log_postvalue - summary["site_avg_log_postvalue"]) / summary["site_std_log_postvalue"]
        area_z_score = (log_postvalue - summary["area_avg_log_postvalue"]) / summary["area_std_log_postvalue"]
        query_write.update(
            {self.model.score: self._calculate_post_score(site_z_score, area_z_score)}, synchronize_session="fetch"
        )

    def _get_log_postvalue_summary(self):
        """Gets summary average and standard deviation of log(postvalue) for posts in site
        and area."""
        postvalue_site = self._get_postvalue_spread(self.query_site)
        postvalue_area = self._get_postvalue_spread(self.query_area)
        return {
            "site_avg_log_postvalue": np.average(np.log(postvalue_site)),
            "area_avg_log_postvalue": np.average(np.log(postvalue_area)),
            "site_std_log_postvalue": np.std(np.log(postvalue_site)),
            "area_std_log_postvalue": np.std(np.log(postvalue_area)),
        }

    def _get_postvalue_spread(self, query):
        """Takes posts within a percentile range of a post's non-normalized score (algorithm defined in
        `self._postvalue_fn`) from a given query. This function's intention is to gather the spread
        of the posts' scores."""
        # Remove posts where price / ft2 <= 0 and ft2 and price / sqrt(ft2) <= 0.
        query = self._filter_query_for_log_calc(query)
        price, ft2, bedrooms = map(
            lambda x: (np.array(x)), zip(*[(post.price, post.ft2, post.bedrooms) for post in query.all()])
        )
        # Gets posts' non-normalized score values (`np.array`) that are within 5% - 95% percentile.
        return self._get_output_within_percentile(
            self._postvalue_fn, np.log, np.sqrt, price, ft2, bedrooms, perc_min=5, perc_max=95
        )

    def _filter_query_for_log_calc(self, query):
        """Updates query to select for self.model.price / self.model.ft2 > 0 and self.model.price /
        sqrt(self.model.ft2) > 0 to allow for logarithmic calculations."""
        return query.filter(
            and_(
                self.model.price / func.sqrt(self.model.ft2) > 0,
                func.log(self.model.price) / func.sqrt(self.model.ft2) > 0,
            )
        )

    @staticmethod
    def _postvalue_fn(log_fn, sqrt_fn, price, ft2, bedrooms):
        """An algorithm to evaluate housing value from price, ft2, and number of bedrooms."""
        return log_fn(log_fn(price) / sqrt_fn(bedrooms)) * price / sqrt_fn(ft2)

class Bedrooms(Score):
    """Calculates score for posts without ft2 attribute. Inherits from `Score`."""

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def write_score(self, query_write):
        """Writes score to posts in `query_write`."""
        # Filter posts to allow for logarithmic operations.
        query_write = self._filter_query_for_log_calc(query_write)
        summary = self._get_log_postvalue_summary()
        log_postvalue = func.log(self._postvalue_fn(func.log, self.model.price, self.model.bedrooms))
        site_z_score = (log_postvalue - summary["site_avg_log_postvalue"]) / summary["site_std_log_postvalue"]
        area_z_score = (log_postvalue - summary["area_avg_log_postvalue"]) / summary["area_std_log_postvalue"]
        query_write.update(
            {self.model.score: self._calculate_post_score(site_z_score, area_z_score)}, synchronize_session="fetch"
        )

    def _get_log_postvalue_summary(self):
        """Gets summary average and standard deviation of log(postvalue) for posts in site
        and area."""
        postvalue_site = self._get_postvalue_spread(self.query_site)
        postvalue_area = self._get_postvalue_spread(self.query_area)
        return {
            "site_avg_log_postvalue": np.average(np.log(postvalue_site)),
            "area_avg_log_postvalue": np.average(np.log(postvalue_area)),
            "site_std_log_postvalue": np.std(np.log(postvalue_site)),
            "area_std_log_postvalue": np.std(np.log(postvalue_area)),
        }

    def _get_postvalue_spread(self, query):
        """Takes posts within a percentile range of a post's non-normalized score (algorithm defined in
        `self._postvalue_fn`) from a given query. This function's intention is to gather the spread
        of the posts' scores."""
        # Filter for bedrooms to be within 0 - 7 range and price > 0.
        query = self._filter_query_for_log_calc(query)
        price, bedrooms = map(lambda x: (np.array(x)), zip(*[(post.price, post.bedrooms) for post in query.all()]))
        # Gets posts' non-normalized score values (`np.array`) that are within 5% - 95% percentile.
        return self._get_output_within_percentile(self._postvalue_fn, np.log, price, bedrooms, perc_min=5, perc_max=95)

    @staticmethod
    def _postvalue_fn(log_fn, price, bedrooms):
        """An algorithm to evaluate housing value from price and number of bedrooms."""
        return (price + (price * (1 - log_fn(bedrooms)))) / 2

    def _filter_query_for_log_calc(self, query):
        """Updates query to select for self.model.price and self.model.bedrooms > 0 AND
        self.model.bedrooms < 8 to allow for logarithmic calculations."""
        return query.filter(and_(self.model.price > 0, self.model.bedrooms > 0, self.model.bedrooms < 8))

def normalize_score(model, query, min_score, max_score):
    """Normalizes score to fall within `min_score` and `max_score`."""
    # Update all model.score value with NoneType to be the average of min and max normalized scores.
    query.filter(model.score == 0).update({model.score: (min_score + max_score) / 2})
    # Get query with all numtypes in model.score.
    query_num = query.filter(model.score != 0)
    min_q_score, max_q_score = get_min_max_scores(query_num)
    query_num.filter(model.score <= min_q_score).update({model.score: min_score})
    query_num.filter(model.score >= max_q_score).update({model.score: max_score})

    # Get query with model.score within min_q_score and max_q_score.
    query_range = query_num.filter(and_(model.score >= min_q_score, model.score <= max_q_score))
    # Equation for normalizing score:
    # (((score - low_score) / (high_score - low_score)) * (2 * max_score)) + min_score
    normalized_score = ((model.score - min_q_score) / (max_q_score - min_q_score)) * (2 * max_score) + min_score
    # Round normalized score to the nearest 5.
    query_range.update({model.score: func.round(normalized_score * 0.2) / 0.2})

def get_min_max_scores(query):
    """To be called after binding scores to posts. Normalizes score to more user friendly values.
    See `normalize_score`. Input query where query.score is all numerics."""
    scores = np.array([post.score for post in query.all()])
    # Takes scores within 0.75% to 99.25% percentile - sets these as absolute min and max, respectively.
    low = np.percentile(scores, 0.75)
    high = np.percentile(scores, 99.25)
    return low, high


text:
from constants import *
import numpy,pickle,dust
import matplotlib.pyplot as plt

dust_perc=input('enter the initial dust percentage, e.g., 95:\n')
print 'percentage of dust=',dust_perc

band=raw_input('what band? 1. 6.35--7.15(kalpha including lyman); 2. 6.35--6.75(excluding lyman)(press enter->default=2):\n')
if band=="":
	band=2
band=int(band)
print 'band=',band,'\n\n'

print "Note: To fit Huanqin's graph, Zehao suggest:\nprecision=1.5 for dust percent=95;\nprecision=2.5 for dust percent=0\nBoth stop at tau=10^11\n"

precision=raw_input('precision(just press enter->default=0.2):\n')
#eV, this number should be <0.5 in order to get high-reso spectra and right centroid (avoid taking conti- spec into account
#    because criteria line < 0.01*maxflux is used)
                ###Zehao's note:0.2 for pltmicurve.py,pltgausscenter.py,plot_ionfrac.py
                ###             1.5 for pltspec.py dust percent=95
                ###             2.5 for pltspec.py dust percent=0
if precision=="":
    precision=0.2
precision=float(precision)
print 'precision=',precision

print 'initializing...'

if band==1:
	dat3=pickle.load(open('data_package/eperionperbin_6.35-7.15_v%s_p%s.pkl' %(version,precision),'rb'))
if band==2:
	dat3=pickle.load(open('data_package/eperionperbin_6.35-6.75_v%s_p%s.pkl' %(version,precision),'rb'))

dat=pickle.load(open('data_package/ionfrac_%s_v%s.pkl' %(str(dust_perc),version),'rb'))

telist=dat['telist']
taulist=dat['taulist']

width=dat3['binwidth']
ecent=dat3['ecent']

tauind=[5,10,100,300,1000,2000,4000,10000,100000]
print "energy bin's width=",width,'keV'
print 'list of centers of each energy bin=\n',ecent

raw_input('press enter to start drawing')

eperionperbin=dat3['eperionperbin']

dust_dblline_center=E_keV #
dust_dblline_yield=yie    #
print 'dust_dblline_center=',dust_dblline_center[1]

print 'energy:\n',ecent-width/2.
for ite,te in enumerate(telist):
	for index in tauind:
		gas_spec=numpy.zeros(len(ecent))
		dust_spec=numpy.zeros(len(ecent))
		for ibin in range(len(ecent)):
			gas_spec[ibin]=sum(eperionperbin[ite,:,ibin]*dat['ionfrac'][ite,index,:])/width
		dust_total_emi=dust.calc_dust_kalpha_emi(te)
		print '\nionfrac rate=',dat['ionfrac'][ite,index,0],'  @Temperature=%s, tau=%s' %(str(te),str(taulist[index]))
		dust_one=dat['ionfrac'][ite,index,0]*dust_total_emi*E_keV[1]*yie[1]/(E_keV[1]*yie[1]+E_keV[2]*yie[2])
		dust_two=dat['ionfrac'][ite,index,0]*dust_total_emi*E_keV[2]*yie[2]/(E_keV[1]*yie[1]+E_keV[2]*yie[2])
		print 'dust_total_emi=',dust_total_emi
		print 'dust_one=',dust_one
		for ii in range(len(ecent)):
			if abs(ecent[ii]-dust_dblline_center[1]) < (width/2.):
				dust_spec[ii]+=dust_one/width
				#print 'find it!',ii
				break
			#else:
				#print 'oh noooooo'
		for ii in range(len(ecent)):
			if abs(ecent[ii]-dust_dblline_center[2]) <= (width/2.):
				dust_spec[ii]+=dust_two/width
				#print '2nd one!',ii
				break
			#else:
				#print 'what?? where is the second one!'
		plt.figure(ite)
		plt.xlabel('energy (keV)')
		plt.ylabel('emissivity $({\mathrm{erg} \mathrm{cm}^3 \mathrm{s}^{-1}} \mathrm{keV}^{-1})$')
		plt.bar(ecent-width/2., gas_spec, width, color='b',alpha=0.7,label='gas_spec',edgecolor='b')
		plt.bar(ecent-width/2., dust_spec,width, color='r',bottom=gas_spec,alpha=0.7,label='dust_spec',edgecolor='r')
		plt.legend(loc=0)

		plt.xlim(6.35,6.75)
		#if dust_perc==0:
			#plt.xlim(6.35,6.75)
			#plt.ylim(0,6*1e-25)
		#if dust_perc==95:
			#plt.xlim(6.3,6.8)
			#plt.ylim(0,1e-25)

		plt.title('tau:10^'+str(round(log10(taulist[index]),1))+' Te:'+str(te)+'keV '+str(dust_perc)+'% dust')
		plt.savefig('results/spec/'+str(te)+'keV_'+str(round(log10(taulist[index]),1))+'logtau_p%s_v%s_' %(str(precision),str(version))+str(dust_perc)+'dust.eps')
		print "energy bin's width=",width,'keV'
		print 'close the graph to continue'
  		plt.show()
		next=raw_input('PRESS ENTER(to draw next tau) \nOR type anything then press enter(to skip to next temperature)\n')
		if next!='':
			break

'''
dat=pickle.load(open('ionfrac_dust.pkl','rb'))

for ite,te in enumerate(telist):
	for itau, tau in enumerate(taulist):
		etmp=dat3['eperionperbin'][ite,:]*dat['ionfrac'][ite,itau,:]
		gasflux[ite,itau]=sum(etmp)
		dustflux[ite,itau]=dat['ionfrac'][ite,itau,0]*dust.calc_dust_kalpha_emi(te)
		totalflux[ite,itau]=gasflux[ite,itau]+dustflux[ite,itau]

ax=fig.add_subplot(122)

for ite,te in enumerate(telist):
	ax.loglog(taulist,gasflux[ite,:],':',label=repr(te))
	ax.loglog(taulist,dustflux[ite,:],'--')
	ax.loglog(taulist,totalflux[ite,:],)

ax.legend(loc=0)
'''
#plt.show()


text:
"""Mixins and core functionality."""
import logging
import os
from collections import namedtuple
from contextlib import closing
from io import BytesIO
from math import ceil
from multiprocessing.pool import ThreadPool

import numpy as np
import requests
from PIL import Image

from helios import CONFIG
from helios.core.request_manager import RequestManager
from helios.core.session import Session
from helios.core.structure import ImageRecord, Record
from helios.utilities import logging_utils, parsing_utils

logger = logging.getLogger(__name__)

class SDKCore(object):
    """
    Core class for Python interface to Helios Core APIs.

    This class must be inherited by any additional Core API classes.
    """
    _max_threads = CONFIG['general']['max_threads']

    def __init__(self, session=None):
        """
        Initialize core API instance.

        If a session has been started prior to initialization it can be used
        via the session input parameter. If this is not used a session will
        be started automatically.

        Args:
            session (helios.Session object, optional): An instance of the
                Session. Defaults to None. If unused a session will be
                created for you.

        """

        # Start session or use custom session.
        if session is None:
            self._session = Session()
        else:
            self._session = session

        # If the session hasn't been started, start it.
        if not self._session.token:
            self._session.start_session()

        # Create request manager to handle all API requests.
        self._request_manager = RequestManager(self._session.token,
                                               pool_maxsize=self._max_threads)

    @property
    def _base_api_url(self):
        return self._session.api_url

    @_base_api_url.setter
    def _base_api_url(self, value):
        raise AttributeError('Access to _base_api_url is restricted.')

    @staticmethod
    def _parse_query_inputs(parameters):
        """
        Create query string from a dictionary of parameters.

        Args:
            parameters (dict):  Key/values to combine into a query string.

        Returns:
            str: Query string.
        """
        parameters_temp = parameters.copy()

        query_str = ''

        # Check for unique case: sensors
        if 'sensors' in parameters_temp:
            query_str += parameters_temp.pop('sensors') + '&'

        # Parse input_dict into a str
        for key, val in parameters_temp.items():
            if val is None:
                continue

            if isinstance(val, (list, tuple)):
                query_str += (str(key) +
                              '=' +
                              ','.join([str(x) for x in val]) +
                              '&')
            else:
                query_str += (str(key) + '=' + str(val) + '&')

        # Remove final ampersand
        query_str = query_str[:-1]

        return query_str

    def _process_messages(self, func, messages):
        n_messages = len(messages)
        logger.info('%s processing %s messages.', func.__name__, n_messages)

        # Create thread pool
        with closing(ThreadPool(min(self._max_threads, n_messages))) as thread_pool:
            results = thread_pool.map(func, messages)

        try:
            n_successful = sum([True for x in results if x.ok])
        except AttributeError:
            pass
        else:
            log_message = '{} out of {} successful'.format(n_successful, n_messages)
            if n_successful == 0:
                logger.error(log_message)
            elif n_successful < n_messages:
                logger.warning(log_message)
            else:
                logger.info(log_message)

        return results

class IndexMixin(object):
    """Mixin for index queries."""

    @logging_utils.log_entrance_exit
    def index(self, **kwargs):
        max_skip = 4000

        limit = int(kwargs.pop('limit', 100))
        skip = int(kwargs.pop('skip', 0))

        # Raise right away if skip is too high.
        if skip > max_skip:
            raise ValueError('skip must be less than the maximum skip value '
                             'of {}. A value of {} was tried.'.format(max_skip, skip))

        # Create the messages up to the maximum skip.
        Message = namedtuple('Message', ['kwargs', 'limit', 'skip'])

        messages = []
        for i in range(skip, max_skip, limit):
            if i + limit > max_skip:
                temp_limit = max_skip - i
            else:
                temp_limit = limit
            messages.append(Message(kwargs=kwargs, limit=temp_limit, skip=i))

        # Process first message.
        initial_resp = self.__index_worker(messages.pop(0))

        # Handle first query failing.
        if not initial_resp.ok:
            logger.error('First query failed. Unable to continue.')
            raise initial_resp.error

        # Get total number of features available.
        try:
            total = initial_resp.content['properties']['total']
        except KeyError:
            total = initial_resp.content['total']

        # Determine number of iterations that will be needed.
        n_queries_needed = int(ceil((total - skip) / float(limit)))
        messages = messages[0:n_queries_needed - 1]

        # Log number of queries required.
        logger.info('%s index queries required for: %s', n_queries_needed, kwargs)

        # If only one query was necessary, return immediately.
        if total <= limit:
            return [initial_resp]

        # Warn the user when truncation occurs. (max_skip is hit)
        if total > max_skip:
            logger.warning('Maximum skip level. Truncated results for: %s',
                           kwargs)

        # Process messages using the worker function.
        results = self._process_messages(self.__index_worker, messages)

        # Put initial query back in list.
        results.insert(0, initial_resp)

        return results

    def __index_worker(self, msg):
        """msg must contain kwargs (search criteria), limit, and skip"""

        params_str = self._parse_query_inputs(msg.kwargs)

        query_str = '{}/{}?{}&limit={}&skip={}'.format(self._base_api_url,
                                                       self._core_api,
                                                       params_str,
                                                       msg.limit,
                                                       msg.skip)

        # Perform query
        try:
            resp = self._request_manager.get(query_str)
        except requests.exceptions.RequestException as e:
            return Record(message=msg, query=query_str, error=e)

        return Record(message=msg, query=query_str, content=resp.json())

class ShowMixin(object):
    """Mixin for show queries"""

    @logging_utils.log_entrance_exit
    def show(self, ids):
        if not isinstance(ids, (list, tuple)):
            ids = [ids]

        # Create messages for worker.
        Message = namedtuple('Message', 'id_')
        messages = [Message(x) for x in ids]

        # Process messages using the worker function.
        results = self._process_messages(self.__show_worker, messages)

        return results

    def __show_worker(self, msg):
        """msg must contain id_"""
        query_str = '{}/{}/{}'.format(self._base_api_url, self._core_api, msg.id_)

        try:
            resp = self._request_manager.get(query_str)
        except requests.exceptions.RequestException as e:
            return Record(message=msg, query=query_str, error=e)

        return Record(message=msg, query=query_str, content=resp.json())

class ShowImageMixin(object):
    """Mixin for show_image queries"""

    @logging_utils.log_entrance_exit
    def show_image(self, id_, data, out_dir=None, return_image_data=False):
        if not isinstance(data, (list, tuple)):
            data = [data]

        # Create messages for worker.
        Message = namedtuple('Message', ['id_', 'data', 'out_dir', 'return_image_data'])
        messages = [Message(id_, x, out_dir, return_image_data) for x in data]

        # Make sure directory exists.
        if out_dir:
            if not os.path.exists(out_dir):
                os.makedirs(out_dir)

        # Process messages using the worker function.
        results = self._process_messages(self.__show_image_worker, messages)

        return results

    def __show_image_worker(self, msg):
        """msg must contain id_, data, out_dir, and return_image_data"""
        query_str = '{}/{}/{}/images/{}'.format(self._base_api_url,
                                                self._core_api,
                                                msg.id_,
                                                msg.data)

        try:
            resp = self._request_manager.get(query_str)
        except requests.exceptions.RequestException as e:
            return ImageRecord(message=msg, query=query_str, error=e)

        # Parse key from url.
        parsed_url = parsing_utils.parse_url(resp.url)
        _, image_name = os.path.split(parsed_url.path)

        # Write image to file.
        if msg.out_dir is not None:
            out_file = os.path.join(msg.out_dir, image_name)
            with open(out_file, 'wb') as f:
                f.write(resp.content)
        else:
            out_file = None

        # Read and return image data.
        if msg.return_image_data:
            # Read image from response.
            img_data = np.asarray(Image.open(BytesIO(resp.content)))
        else:
            img_data = None

        return ImageRecord(message=msg, query=query_str, name=image_name,
                           content=img_data, output_file=out_file)

max_stars_repo_name: harris-helios/helios-sdk-python

max_issues_repo_name: harris-helios/helios-sdk-python

max_forks_repo_name: harris-helios/helios-sdk-python


text:
# Copyright 2019 Carnegie Mellon University
#
# Based on:
# https://github.com/tensorflow/models/blob/master/research/object_detection/object_detection_tutorial.ipynb

import numpy as np
import tensorflow as tf
import pickle
import time
import heapq

from distutils.version import StrictVersion
from PIL import Image

# From https://github.com/tensorflow/models/blob/master/research/object_detection/utils/visualization_utils.py#L632
MIN_SCORE_THRESH = 0.5
JACCARD_THRESHOLD = 0.5

if StrictVersion(tf.__version__) < StrictVersion('1.9.0'):
  raise ImportError('Please upgrade your TensorFlow installation to v1.9.* or later!')

def calc_iou(box_a, box_b):
  # From https://github.com/georgesung/ssd_tensorflow_traffic_sign_detection/
  # blob/e5f32413fb56279d5b6fb51a243ca06e5c567dce/data_prep.py#L9
  x_overlap = max(0, min(box_a[2], box_b[2]) - max(box_a[0], box_b[0]))
  y_overlap = max(0, min(box_a[3], box_b[3]) - max(box_a[1], box_b[1]))
  intersection = x_overlap * y_overlap

  area_box_a = (box_a[2] - box_a[0]) * (box_a[3] - box_a[1])
  area_box_b = (box_b[2] - box_b[0]) * (box_b[3] - box_b[1])
  union = area_box_a + area_box_b - intersection

  iou = intersection / union
  return iou

# # Model preparation

# ## Variables
#
# Any model exported using the `export_inference_graph.py` tool can be loaded here simply by changing `PATH_TO_FROZEN_GRAPH` to point to a new .pb file.
#
# By default we use an "SSD with Mobilenet" model here. See the [detection model zoo](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/detection_model_zoo.md) for a list of other models that can be run out-of-the-box with varying speeds and accuracies.

# ## Load a (frozen) Tensorflow model into memory.

# In[10]:

detection_graph = tf.Graph()
with detection_graph.as_default():
  od_graph_def = tf.GraphDef()
  with tf.gfile.GFile('/home/ubuntu/Cross-Safe/models/research/new_splitted_data/exported_graphs/frozen_inference_graph.pb', 'rb') as fid:
    serialized_graph = fid.read()
    od_graph_def.ParseFromString(serialized_graph)
    tf.import_graph_def(od_graph_def, name='')

# ## Helper code

# In[12]:

def load_image_into_numpy_array(image):
  (im_width, im_height) = image.size
  return np.array(image.getdata()).reshape(
      (im_height, im_width, 3)).astype(np.uint8)

# # Detection

# In[13]:

# For the sake of simplicity we will use only 2 images:
# image1.jpg
# image2.jpg
# If you want to test the code with your images, just add path to the images to the TEST_IMAGE_PATHS.
with open('/home/ubuntu/Cross-Safe/models/research/new_splitted_data/holdout_files.p', 'rb') as f:
    filenames = pickle.load(f)
TEST_IMAGE_PATHS = [
    '/home/ubuntu/Cross-Safe/data/new_splitted_data/{}.JPEG'.format(filename)
    for filename in filenames
]

# In[14]:

def write_inference_and_ground_truth(graph):
  with graph.as_default():
    with tf.Session() as sess:
      # Get handles to input and output tensors
      ops = tf.get_default_graph().get_operations()
      all_tensor_names = {output.name for op in ops for output in op.outputs}
      tensor_dict = {}
      for key in [
          'num_detections', 'detection_boxes', 'detection_scores',
          'detection_classes'
      ]:
        tensor_name = key + ':0'
        if tensor_name in all_tensor_names:
          tensor_dict[key] = tf.get_default_graph().get_tensor_by_name(
              tensor_name)

      multiple_boxes = []
      wrong_class = []
      no_box = []

      correct_single_box = 0
      total = 0

      jaccard_total = 0
      heap = []

      dont_walk_labels = []
      dont_walk_scores = []

      walk_labels = []
      walk_scores = []

      total_predictions = 0

      for image_path in TEST_IMAGE_PATHS:
        total += 1

        image = Image.open(image_path)
        image = image.convert('RGB')
        # the array based representation of the image will be used later in order to prepare the
        # result image with boxes and labels on it.
        image_np = load_image_into_numpy_array(image)

        image_tensor = tf.get_default_graph().get_tensor_by_name('image_tensor:0')

        with open('times.txt', 'a+') as timefile:
          # Run inference
          start = time.time()
          output_dict = sess.run(tensor_dict,
                                 feed_dict={image_tensor: np.expand_dims(image_np, 0)})
          end = time.time()
          timefile.write('{}\n'.format(end - start))

          # num_detections = int(output_dict['num_detections'][0])
          detection_classes = output_dict[
            'detection_classes'][0].astype(np.uint8)
          detection_boxes = output_dict['detection_boxes'][0]
          detection_scores = output_dict['detection_scores'][0]

          label_without_jpeg = image_path[:-5]
          with open('{}.txt'.format(label_without_jpeg)) as f:
            true_class = int(f.readline())
            ground_truth_bounding_box_file = [
              int(num)
              for num in f.readline().split()
            ]

            # From https://github.com/tensorflow/models/blob/master/research/object_detection/utils/visualization_utils.py#L721
            ground_truth_bounding_box = [
              ground_truth_bounding_box_file[1] / 300,
              ground_truth_bounding_box_file[0] / 400,
              ground_truth_bounding_box_file[3] / 300,
              ground_truth_bounding_box_file[2] / 400,
            ]

            found_true_class = False
            for detection_score, detection_box, detection_class in (
                zip(detection_scores, detection_boxes, detection_classes)):
              if detection_score > 0.5:
                total_predictions += 1

                label_value = 0
                correct_box = calc_iou(detection_box, ground_truth_bounding_box) > JACCARD_THRESHOLD
                if (true_class == detection_class and correct_box):
                  found_true_class = True
                  label_value = 1
                else:
                  label_value = 0

                if detection_class == 1:
                  dont_walk_labels.append(label_value)
                  dont_walk_scores.append(detection_score)
                else:
                  walk_labels.append(label_value)
                  walk_scores.append(detection_score)

            if not found_true_class:
              if detection_class == 1:
                dont_walk_labels.append(1)
                dont_walk_scores.append(0)
              else:
                walk_labels.append(1)
                walk_scores.append(0)

            if detection_scores[1] > MIN_SCORE_THRESH:
              multiple_boxes.append(image_path)
            else:
              if detection_scores[0] <= MIN_SCORE_THRESH:
                no_box.append(image_path)
              elif true_class != detection_classes[0]:
                wrong_class.append(image_path)

              else:
                correct_single_box += 1

                inference_bounding_box = detection_boxes[0]

                single_jaccard = calc_iou(inference_bounding_box, ground_truth_bounding_box)
                heapq.heappush(heap, (single_jaccard, image_path))
                jaccard_total += single_jaccard

      ordered_by_jaccard = []
      while len(heap) > 0:
        popped = heapq.heappop(heap)[1]
        ordered_by_jaccard.append(popped)
      print('ordered by jaccard', ordered_by_jaccard)

      print('multiple boxes', len(multiple_boxes), multiple_boxes)
      print('wrong class', len(wrong_class), wrong_class)
      print('no box', len(no_box), no_box)

      print('total', total)
      print('correct single box', correct_single_box)
      print('Average Jaccard for correct single box', jaccard_total / correct_single_box)

      print('total predictions', total_predictions)

      with open('dont_walk_labels.p', 'wb') as pickle_file:
        pickle.dump(dont_walk_labels, pickle_file)

      with open('dont_walk_scores.p', 'wb') as pickle_file:
        pickle.dump(dont_walk_scores, pickle_file)

      with open('walk_labels.p', 'wb') as pickle_file:
        pickle.dump(walk_labels, pickle_file)

      with open('walk_scores.p', 'wb') as pickle_file:
        pickle.dump(walk_scores, pickle_file)

      from sklearn.metrics import average_precision_score
      walk_average_precision = average_precision_score(walk_labels, walk_scores)
      dont_walk_average_precision = average_precision_score(dont_walk_labels, dont_walk_scores)

      total_precision = walk_average_precision + dont_walk_average_precision
      mAP = total_precision / 2

      print('mAP', mAP)

write_inference_and_ground_truth(detection_graph)

max_stars_repo_name: cmusatyalab/cmu-cross-safe

max_issues_repo_name: cmusatyalab/cmu-cross-safe

max_forks_repo_name: cmusatyalab/cmu-cross-safe


text:
import math as m
import numpy as np

def E2R(Ry, Rx, Rz):
    '''Combine Euler angles to the rotation matrix (right-hand)

        Inputs:
            Ry, Rx, Rz : rotation angles along  y, x, z axis
                         only has Ry in the KITTI dataset
        Returns:
            3 x 3 rotation matrix

    '''
    R_yaw = np.array([[ m.cos(Ry), 0 ,m.sin(Ry)],
                      [ 0,         1 ,     0],
                      [-m.sin(Ry), 0 ,m.cos(Ry)]])
    R_pitch = np.array([[1, 0, 0],
                        [0, m.cos(Rx), -m.sin(Rx)],
                        [0, m.sin(Rx), m.cos(Rx)]])
    #R_roll = np.array([[[m.cos(Rz), -m.sin(Rz), 0],
    #                    [m.sin(Rz), m.cos(Rz), 0],
    #                    [ 0,         0 ,     1]])
    return (R_pitch.dot(R_yaw))

def project_corns(corns, intrins):
    '''
    corns: 3*8
    intrins: 3*4

    return: 8*2
    '''
    corns = np.concatenate([corns, np.ones((1,8))], axis=0)
    proj_points = np.dot(intrins, corns)
    proj_points[0,:] = proj_points[0,:]/proj_points[2,:]
    proj_points[1,:] = proj_points[1,:]/proj_points[2,:]
    proj_points = np.transpose(proj_points[:2,:])
    return proj_points

max_stars_repo_name: chencgln/glimg

max_issues_repo_name: chencgln/glimg

max_forks_repo_name: chencgln/glimg


text:
import math

import numpy as np
import torch
import torch.nn as nn
from torch.autograd import Variable

'''
The basic components to be used in our network (include GCN and TCN).
'''

def conv_branch_init(conv, branches):
    weight = conv.weight
    n = weight.size(0)
    k1 = weight.size(1)
    k2 = weight.size(2)
    nn.init.normal_(weight, 0, math.sqrt(2. / (n * k1 * k2 * branches)))
    nn.init.constant_(conv.bias, 0)

def conv_init(conv):
    nn.init.kaiming_normal_(conv.weight, mode='fan_out')
    nn.init.constant_(conv.bias, 0)

def bn_init(bn, scale):
    nn.init.constant_(bn.weight, scale)
    nn.init.constant_(bn.bias, 0)

class unit_tcn(nn.Module):
    def __init__(self, in_channels, out_channels, kernel_size=9, stride=1):
        super(unit_tcn, self).__init__()
        pad = int((kernel_size - 1) / 2)
        self.conv = nn.Conv2d(in_channels, out_channels, kernel_size=(kernel_size, 1), padding=(pad, 0),
                              stride=(stride, 1))

        self.bn = nn.BatchNorm2d(out_channels)
        self.relu = nn.ReLU()
        conv_init(self.conv)
        bn_init(self.bn, 1)

    def forward(self, x):
        x = self.bn(self.conv(x))
        return x

class unit_gcn(nn.Module):
    def __init__(self, in_channels, out_channels, A, coff_embedding=4, num_subset=3):
        super(unit_gcn, self).__init__()
        inter_channels = out_channels // coff_embedding
        self.inter_c = inter_channels
        self.PA = nn.Parameter(torch.from_numpy(A.astype(np.float32)))
        nn.init.constant_(self.PA, 1e-6)
        self.A = Variable(torch.from_numpy(A.astype(np.float32)), requires_grad=False)
        self.num_subset = num_subset

        self.conv_a = nn.ModuleList()
        self.conv_b = nn.ModuleList()
        self.conv_d = nn.ModuleList()
        for i in range(self.num_subset):
            self.conv_a.append(nn.Conv2d(in_channels, inter_channels, 1))
            self.conv_b.append(nn.Conv2d(in_channels, inter_channels, 1))
            self.conv_d.append(nn.Conv2d(in_channels, out_channels, 1))

        if in_channels != out_channels:
            self.down = nn.Sequential(
                nn.Conv2d(in_channels, out_channels, 1),
                nn.BatchNorm2d(out_channels)
            )
        else:
            self.down = lambda x: x

        self.bn = nn.BatchNorm2d(out_channels)
        self.soft = nn.Softmax(-2)
        self.relu = nn.ReLU()

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                conv_init(m)
            elif isinstance(m, nn.BatchNorm2d):
                bn_init(m, 1)
        bn_init(self.bn, 1e-6)
        for i in range(self.num_subset):
            conv_branch_init(self.conv_d[i], self.num_subset)

    def forward(self, x):
        N, C, T, V = x.size()
        A = self.A.cuda(x.get_device())
        A = A + self.PA

        y = None
        for i in range(self.num_subset):
            A1 = self.conv_a[i](x).permute(0, 3, 1, 2).contiguous().view(N, V, self.inter_c * T)
            A2 = self.conv_b[i](x).view(N, self.inter_c * T, V)
            A1 = self.soft(torch.matmul(A1, A2) / A1.size(-1))  # N V V
            A1 = A1 + A[i]
            A2 = x.view(N, C * T, V)
            z = self.conv_d[i](torch.matmul(A2, A1).view(N, C, T, V))
            y = z + y if y is not None else z

        y = self.bn(y)
        y += self.down(x)
        return self.relu(y)

max_stars_repo_name: Yiran-Zhu/GCHB

max_issues_repo_name: Yiran-Zhu/SkeleGraph

max_forks_repo_name: Yiran-Zhu/SkeleGraph


text:
"""
Activation rule for layer q:
    v[q][t+1] = tanh(sum_(p,q,r) s[p]*W[p].dot(v[r][t])
Learning rule for pathway p to q from r:
    W[p] += l[p] * (arctanh(v[q][t+1]) - W[p].dot(v[r][t])) * v[r][t].T / N
"""
import numpy as np
import torch as tr
import torch.nn as nn
from codec import Codec
from controller import Controller

class GatedHebbianUnit(object):
    def __init__(self, layer_sizes, pathways, controller, codec, batch_size=1, plastic=[]):
        """
        layer_sizes[k] (dict): size of layer k
        pathways[p]: (destination, source) for pathway p
        controller: dict of layer activity -> s,l gate dicts
        batch_size: number of examples to process at a time
        """
        super(GatedHebbianUnit, self).__init__()
        self.layer_sizes = layer_sizes
        self.pathways = pathways
        self.controller = controller
        self.codec = codec
        self.batch_size = batch_size
        self.plastic = plastic
        self.WL = {p: tr.zeros(batch_size, layer_sizes[q], 1)
            for p, (q,r) in pathways.items()}
        self.WR = {p: tr.zeros(batch_size, 1, layer_sizes[r])
            for p, (q,r) in pathways.items()}
        self.v = {t:
            {q: tr.zeros(batch_size, size)
                for q, size in layer_sizes.items()}
            for t in [-1, 0]}
        self.h = {-1: tr.zeros(1, batch_size, controller.hidden_size)}
        self.al = {}
        self.pl = {}
        self.ac = {}
        self.pc = {}

    def clone(self):
        # copies associative weight matrices and initial activity
        # assumes no ticks have been called yet
        ghu = GatedHebbianUnit(
            layer_sizes = self.layer_sizes,
            pathways = self.pathways,
            controller = self.controller,
            codec = self.codec,
            batch_size = self.batch_size,
            plastic = self.plastic)
        ghu.v = {t:
            {q: self.v[t][q].clone().detach() for q in self.layer_sizes.keys()}
            for t in [-1, 0]}
        ghu.WL = {p: self.WL[p].clone().detach() for p in self.pathways.keys()}
        ghu.WR = {p: self.WR[p].clone().detach() for p in self.pathways.keys()}
        return ghu

    def rehebbian(self, WL, WR, x, y):
        r = self.codec.rho
        n = x.shape[-1] * r**2
        g = 0.5*(np.log(1. + r) - np.log(1. - r)) / r # formula for arctanh(r)
        dWL = g*y.unsqueeze(2) - tr.matmul(WL, tr.matmul(WR, x.unsqueeze(2)))
        dWR = x.unsqueeze(1) / n
        # WL, WR = tr.cat((WL, dWL), dim=2), tr.cat((WR, dWR), dim=1)
        # return WL, WR
        return dWL, dWR

    def tick(self, detach=True, choices=None, verbose=0):
        # if provided, choices get passed to controller

        # Controller
        if verbose > 0: print("  Controller forward pass...")
        t = len(self.al)
        _, choices, likelihoods, self.h[t] = self.controller.act(
            self.v[t] if not detach else
                {q: v.clone().detach() for q, v in self.v[t].items()},
            self.h[t-1], choices)
        self.ac[t], self.pc[t] = choices
        self.al[t], self.pl[t] = likelihoods

        # Associative recall
        if verbose > 0: print("  Associative recall...")
        self.v[t+1] = {}
        WL, WR, rv = {}, {}, {} # accumulate data from different gate choices in different episodes
        if verbose > 1: print("   Selecting pathways for batch matmul...")
        for q, qsz in self.layer_sizes.items():
            # Get largest number of updates to an incoming pathway to layer q
            qt = max([self.WL[p].shape[-1] for p in self.controller.incoming[q]])
            # Get largest source layer size in the pathways to layer q
            rsz = max([self.layer_sizes[self.pathways[p][1]] for p in self.controller.incoming[q]])
            # Init sufficiently large pathway and source layer tensors
            WL[q] = tr.zeros(self.batch_size, qsz, qt)
            WR[q] = tr.zeros(self.batch_size, qt, rsz)
            rv[q] = tr.zeros(self.batch_size, rsz)
            # Process all possible incoming pathways
            for i, p in enumerate(self.controller.incoming[q]):
                # Get source layer and number of weight updates to current pathway
                _, r = self.pathways[p] # source layer
                pt = self.WL[p].shape[-1] # number of updates
                # Get mask of episodes where current pathway was ungated
                b = (self.ac[t][q][0] == i).squeeze()
                # Store weights and source activity from respective episodes
                WL[q][b, :, :pt], WR[q][b, :pt, :] = self.WL[p][b], self.WR[p][b]
                rv[q][b, :self.layer_sizes[r]] = self.v[t][r][b]
        if verbose > 1: print("   Performing matmul...")
        for q in self.layer_sizes.keys():
            self.v[t+1][q] = tr.tanh(
                tr.matmul(WL[q], tr.matmul(WR[q], rv[q].unsqueeze(2)))).squeeze(2)

        # Associative learning
        if verbose > 0: print("  Associative learning...")
        for p in self.pathways.keys():
            q, r = self.pathways[p]
            dWL = tr.zeros(self.batch_size, self.layer_sizes[q], 1)
            dWR = tr.zeros(self.batch_size, 1, self.layer_sizes[r])
            # Select out batch elements where pathway p learns
            if p in self.plastic:
                i = self.plastic.index(p)
                b = (self.pc[t][0,:,i] == 1)
                if b.sum() > 0:
                    dWL[b], dWR[b] = self.rehebbian(
                        self.WL[p][b], self.WR[p][b], self.v[t][r][b], self.v[t][q][b])
            self.WL[p] = tr.cat((self.WL[p], dWL), dim=2)
            self.WR[p] = tr.cat((self.WR[p], dWR), dim=1)

    def associate(self, associations, check=True):
        T = len(self.WL)-1
        for p, s, t in associations:
            q, r = self.pathways[p]
            x = tr.repeat_interleave(
                self.codec.encode(r, s).view(1,-1),
                self.batch_size, dim=0)
            y = tr.repeat_interleave(
                self.codec.encode(q, t).view(1,-1),
                self.batch_size, dim=0)
            dWL, dWR = self.rehebbian(self.WL[p], self.WR[p], x, y)
            self.WL[p] = tr.cat((self.WL[p], dWL), dim=2)
            self.WR[p] = tr.cat((self.WR[p], dWR), dim=1)

        if not check: return
        for p,s,t in associations:
            q,r = self.pathways[p]
            Wv = tr.matmul(self.WL[p][0], tr.matmul(self.WR[p][0], self.codec.encode(r, s).view(-1,1)))
            assert(self.codec.decode(q, Wv.squeeze()) == t)

    def fill_layers(self, symbol):
        for t in self.v:
            for k in self.layer_sizes.keys():
                self.v[t][k] = tr.repeat_interleave(
                    self.codec.encode(k, symbol).view(1,-1),
                    self.batch_size, dim=0)

    def run(self, episode_duration, inputs, targets, reward, choices=None, verbose=0):
        """
        Run on a batch of episodes and assess rewards
            episode_duration: number of ticks in each episode
            inputs[b][t]: input symbol at time t in episode b
            targets[b]: target output sequence for episode b
            reward(self, targets, outputs)[t]: reward at time t for given target/output sequences
            choices: if provided, overwrites randomly sampled choices
                choices[t][0][q]: pathway selected for recall in layer q at time t
                choices[t][1][n]: nth pathway with learning on at time t
        """

        # Encode choices
        if choices is None: choices = [None]*episode_duration
        else: choices = [(
            {q: tr.stack([
                tr.tensor([self.controller.incoming[q].index(p)])
                for b in range(self.batch_size)]).reshape(1,self.batch_size,1)
                for q,p in ac.items()},
            tr.tensor([
                [(p in pc) for p in self.plastic]
                for b in range(self.batch_size)], dtype=tr.float32)
                .reshape(1,self.batch_size,len(self.plastic)))
            for (ac, pc) in choices]

        if verbose > 0: print("Encoding episode inputs...")
        encoded = {}
        for t in range(len(inputs[0])):
            if verbose > 1: print(" t=%d..." % t)
            encoded[t] = tr.stack([
                self.codec.encode("rinp", inputs[b][t])
                for b in range(self.batch_size)])

        if verbose > 0: print("Running episodes...")
        for t in range(episode_duration):
            if verbose > 1: print(" t=%d..." % t)
            if t < len(encoded): self.v[t]["rinp"] = encoded[t] # insert input
            self.tick(choices=choices[t], verbose=verbose-1) # tick

        if verbose > 0: print("Decoding outputs...")
        outputs = np.empty((self.batch_size, episode_duration), dtype=str)
        for t in range(episode_duration):
            if verbose > 1: print(" t=%d..." % t)
            outputs[:,t] = [
                self.codec.decode("rout", self.v[t+1]["rout"][b,:])
                for b in range(self.batch_size)]

        # Assess rewards
        if verbose > 0: print("Assessing reward...")
        rewards = np.array([
            reward(self, targets[b], outputs[b])
            for b in range(self.batch_size)])

        # Summarize first episode
        for t in range(episode_duration+1):
            if verbose > 2: print(" Step %d" % t)
            if verbose > 2: print(" layers: ",
                {k: self.codec.decode(k, self.v[t][k][0]) for k in self.layer_sizes.keys()})
            if t == episode_duration: break
            if verbose > 2: print(" choices: ",
                {q: self.controller.incoming[q][ac[0,0].item()] for q,ac in self.ac[t].items()},
                [pc[0].item() for pc in self.pc[t] if len(self.plastic) > 0])
            if verbose > 2: print(" likelihoods: ",
                {q: "%.3f" % al[0,0].item() for q,al in self.al[t].items()},
                ["%.3f" % pl[0].item() for pl in self.pl[t] if len(self.plastic) > 0])
            if verbose > 2: print(" Reward=%f)" % rewards[0,t])
        if verbose > 2: print(" Net reward=%f)" % rewards[0].sum())

        # return results
        return outputs, rewards

def default_initializer(register_names, symbols):
    pathways = {
        q+"<"+r: (q,r)
        for q in register_names
        for r in register_names}
    associations = [(p,a,a)
        for p in list(pathways.keys())
        for a in symbols]
    return pathways, associations

def turing_initializer(register_names, num_addresses):

    # defaults
    symbols = list(map(str, range(num_addresses)))
    pathways, associations = default_initializer(["m"] + register_names, symbols)

    # tape shifts
    pathways.update({k: ("m","m") for k in ["inc-m","dec-m"]})
    associations += [
        (k, str(a), str((a+x) % num_addresses))
        for k,x in [("inc-m",1), ("dec-m",-1)]
        for a in range(num_addresses)]

    return pathways, associations

if __name__ == "__main__":

    layer_sizes = {"r0": 3, "r1":3}
    pathways = {0:("r0","r0"), 1:("r1","r0"), 2:("r1","r1")}
    hidden_size = 5
    batch_size = 2

    codec = Codec(layer_sizes, "01")
    controller = Controller(layer_sizes, pathways, hidden_size)
    ghu = GatedHebbianUnit(layer_sizes, pathways, controller, codec, batch_size=batch_size)

    ghu.associate([
        (0, "0", "0"),
        (1, "0", "0"),
        (2, "0", "0"),
        (0, "1", "1"),
        (1, "1", "1"),
        (2, "1", "1"),
        ])

    # a = codec.encode("r0","0")
    # b = codec.encode("r0","1")
    # print(a)
    # x = codec.decode("r0",a)
    # print(x == "0")
    # print(x == "1")
    # y = codec.decode("r0",b)
    # print(y == "0")
    # print(y == "1")
    # print(codec.parameters())

    for t in [-1,0]:
        for k in ["r0","r1"]:
            ghu.v[t][k] = tr.repeat_interleave(
                codec.encode(k, str(0-t)).view(1,-1),
                batch_size, dim=0)
            # ghu.v[-1]["r0"] = codec.encode("r0", str(1))
            # ghu.v[-1]["r1"] = codec.encode("r1", str(1))
            # ghu.v[0]["r0"] = codec.encode("r0", str(0))
            # ghu.v[0]["r1"] = codec.encode("r1", str(0))
    ghu.tick(num_steps=2)

    # e = ghu.g[len(ghu.g)-1].sum()
    # e.backward()

    # print("codec")
    # print(codec)
    # for p in codec.parameters():
    #     print(p.grad)
    # print("ctrl")
    # print(controller)
    # for p in controller.parameters():
    #     print(p.grad)

    # pathways, associations = turing_initializer(["r0","r1"], 3)
    # for p in pathways:
    #     print(p, pathways[p])
    #     for p1,a,b in associations:
    #         if p1 == p:
    #             print(" %s -> %s"% (a,b))

max_stars_repo_name: garrettkatz/ghu

max_issues_repo_name: garrettkatz/ghu

max_forks_repo_name: garrettkatz/ghu


text:
import math
from typing import Optional
import numpy as np
import gym
from gym import spaces
from gym.utils import seeding
from statistics import mean

class MechanismDesigner(gym.Env):

  def __init__(self, n, m):
    super(MechanismDesigner, self).__init__()
    self.action_space = spaces.Box(low=np.array([-10]),
                                   high=np.array([10]),
                                   shape=(1,),
                                   dtype=np.float32)
    self.observation_space = spaces.Box(low=np.array([0,0]),
                                       high=np.array([1,1]),
                                       shape=(2,),
                                       dtype=np.float32)
    self.n = n
    self.m = m
    self.episode_threshold = 10
    self.step_count = 0
    self.state = 0
    self.inner_layer_total_time_steps = 10000

  def step(self, action):
    self.step_count = self.step_count + 1
    self.state = 0
    reward = self.reward(action)
    done = bool(self.step_count > self.episode_threshold)

    return np.array([self.state]), reward, done, {}

  def reward(self,action):
    reward = 0
    if action[0] > 0 :
      welfare,variance = self.simulate_inner_layer(action[0])
      if (welfare > 0.5) & (variance < 0.5) :
        reward = 50
      elif (welfare > 0.5) & (variance > 0.5) :
        reward = 15
      elif (welfare < 0.5) & (variance < 0.5) :
        reward = 15
      else
        reward = 0
    return reward

  def simulate_inner_layer(self,mechanism_parameter):
    inner_layer_env = MultiAgentEnv.create_environment(self.n,self.m,mechanism_parameter)
    welfare = maddpg.train(inner_layer_env, time_step = self.inner_layer_total_time_steps)
    return welfare, variance

  def reset(self, seed: Optional[int] = None):
      self.step_count=0
      if seed is not None or self.np_random is None:
          self.np_random, seed = seeding.np_random(seed)
      self.state = 0
      return np.array([self.state], dtype=np.float32)

  def render(self, mode='human', close=False):
      # TODO : PettingZOO MPE Render
      pass

  def close(self):
      pass

def simulate_inner_layer(mechanism_parameter,n,m):
  from multiagent.scenrios import inner_layer
  inner_layer_env = inner_layer.create_environment(n,m,mechanism_parameter)
  welfare,variance = maddpg.train(inner_layer_env, time_step = 100000)
  return welfare ,variance


text:
import numpy as np
import pathlib, sys
import pressiotools.linalg as ptla
from pressiotools.io.array_read import *
from pressiotools.io.array_write import *

# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# all the _fnc are implementation details
# check the end for the actual publicly exposed function
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

def _processYamlDictionaryForBasis(yamlDic):
  nCols        = yamlDic["num-columns"]
  fileRootName = yamlDic["file-root-name"]
  fileFmt      = yamlDic["format"]
  if fileFmt=="binary":
    isBinary=True
  elif fileFmt=="ascii":
    isBinary=False
  else:
    sys.exit("Unsupported file format: format must be 'binary' or 'ascii'.")

  return {"fileRootName"   : fileRootName,
          "isBinary"       : isBinary,
          "nCols"          : nCols}

#-----------------------------------------------------------------
def _processYamlDictionary(yamlDic):
  # dataDir is not an entry specified in the real yaml file
  # so you won't find it inside the inputs-templates/...template.yaml
  # because it is passed to driver script as cmd line arg
  # and inserted in the dic by the driver script
  dataDir      = yamlDic["dataDir"]

  dofsPerMnode = yamlDic["FullMeshInfo"]["dofs-per-mesh-node"]

  # residual basis info
  residBasisDic = _processYamlDictionaryForBasis(yamlDic["ResidualBasis"])

  # weighting matrix info
  outDir             = yamlDic["WeightingMatrix"]["outDir"]
  weightMatKind      = yamlDic["WeightingMatrix"]["kind"]
  sampMatIndFileName = yamlDic["WeightingMatrix"]["sample-mesh-indices-filename"]
  fileFmt            = yamlDic["WeightingMatrix"]["format"]
  if fileFmt=="binary":
    isBinary=True
  elif fileFmt=="ascii":
    isBinary=False
  else:
    sys.exit("Unsupported file format: format must be 'binary' or 'ascii'.")

  return {"dofsPerMnode"       : dofsPerMnode,
          "ResidualBasis"      : residBasisDic,
          "weightMatKind"        : weightMatKind,
          "sampMatIndFileName" : sampMatIndFileName,
          "isBinary"           : isBinary,
          "outDir"             : outDir,
          "dataDir"            : dataDir}

#-----------------------------------------------------------------
def _checkInputsValidity(dic):
  if dic["dofsPerMnode"] < 1:
    sys.exit("unsupported input: ResidualBasis/dofs-per-mesh-node needs to be greater than 0")

  if dic["ResidualBasis"] and dic["ResidualBasis"]["nCols"] < 1:
    sys.exit("unsupported input: ResidualBasis/nCols needs to be greater than 0")

  if dic["weightMatKind"] != "gnat":
    sys.exit("In yaml file, you set: *{}*, but LSPG Weighting only supports GNAT".format(dic["weightMatKind"]))

#-----------------------------------------------------------------
def _readData(dataDir, dic, comm=None):
  rootFileName = dic["fileRootName"]
  nCols        = dic["nCols"]
  isBinary     = dic["isBinary"]

  if comm is not None and comm.Get_size() > 1:
    fileName = dataDir + "/" + rootFileName
    psi = read_array_distributed(comm, fileName, nCols, isBinary)
    return psi
  else:
    fileName = "{}.{}.{:0{width}d}".format(rootFileName,1,0,width=1)
    psi0 = read_array(dataDir + "/" + fileName, nCols, isBinary)
    psi  = ptla.MultiVector(psi0)
    return psi

#-----------------------------------------------------------------
def _writeResultsToFile(outDir, weighting, isBinary=True, comm=None):
  nullComm = True if comm is None else False
  rank   = 0 if nullComm else comm.Get_rank()

  fileName = outDir + "/lspgWeighting"
  if isBinary:
    fileName+=".bin"
  else:
    fileName+=".txt"

  if nullComm:
    write_array(weighting, fileName, isBinary)
  else:
    write_array_distributed(comm, weighting, fileName, isBinary)

#-----------------------------------------------------------------
def _getSamplingMatrixLocalInds(sampleMeshGlobalInds, dofsPerMnode, matrix, comm=None):
  # assume that global indices are:
  # i_rank * n_node_per_rank + i_array_local, i_rank < n_ranks

  nullComm = True if comm is None else False
  rank   = 0 if nullComm else comm.Get_rank()
  nRanks = 1 if nullComm else comm.Get_size()

  if len(sampleMeshGlobalInds) == 0:
    return []
  else:
    myNumRows = matrix.extentLocal(0)
    myMinRowGid = matrix.minRowGidLocal()
    myMaxRowGid = matrix.maxRowGidLocal()

    # determine my global mesh indices
    myNumMeshNodes   = int(myNumRows/dofsPerMnode)
    myMinMeshGid     = int(myMinRowGid/dofsPerMnode)
    myGlobalMeshInds = myMinMeshGid + np.arange(myNumMeshNodes)

    if myMinMeshGid < 0: sys.exit("Global Indices must be positive")

    sampleMeshLocalInds = np.intersect1d(sampleMeshGlobalInds, myGlobalMeshInds) - myMinMeshGid

    torep = sampleMeshLocalInds*dofsPerMnode+myMinRowGid
    shift = np.tile(np.arange(dofsPerMnode), len(sampleMeshLocalInds))
    rowsLocalInds = np.repeat(torep, dofsPerMnode) + shift - myMinRowGid
    return rowsLocalInds.tolist()

#-----------------------------------------------------------------
def _computeLspgWeightingImpl(psi,
                              dofsPerNode,
                              sampleMeshGlobaIndices,
                              comm):

  nullComm = True if comm is None else False
  rank     = 0 if nullComm else comm.Get_rank()
  nRanks   = 1 if nullComm else comm.Get_size()

  rowsLocalInds = _getSamplingMatrixLocalInds(sampleMeshGlobaIndices,\
                                              dofsPerNode,\
                                              psi,\
                                              comm)
  # here we compute:
  # ((Z*psi)^+)^T psi^T psi (Z*psi)^+

  # compute (Z*psi)^+
  zpsi = ptla.MultiVector(np.asfortranarray(psi.data()[rowsLocalInds, :]))
  pinvObj = ptla.PseudoInverse()
  pinvObj.compute(zpsi)

  # compute B = psi^T psi, where B is replicated on each rank
  B = np.zeros((psi.extentLocal(1), psi.extentLocal(1)), order='F')
  ptla.selfTransposeSelf(1., psi, B)

  # compute C = ((Z*psi)^+)^T psi^T psi
  C = ptla.MultiVector(pinvObj.applyTranspose(B))

  # compute target result
  zpsiT = ptla.MultiVector(pinvObj.viewTransposeLocal())
  D = np.zeros((C.extentLocal(0), zpsiT.extentGlobal(0)), order='F')
  ptla.product('N','T', 1., C, zpsiT, 0., D)

  return None if len(rowsLocalInds)==0 else D

#-----------------------------------------------------------------
def _computeLspgWeightingReadYaml(comm, yaml_in):
  # process user inputs
  dic = _processYamlDictionary(yaml_in)

  # check that inputs are valid, exit with error otherwise
  _checkInputsValidity(dic)

  # read residual basis matrix
  psi = _readData(dic["dataDir"], dic["ResidualBasis"], comm)

  # Read sample mesh GLOBAL indices
  smGidFilePath = dic["dataDir"]+"/"+dic["sampMatIndFileName"]
  smGlobalIndices = np.loadtxt(smGidFilePath, dtype=np.int)

  dofsPerNode = dic["dofsPerMnode"]
  weighting = _computeLspgWeightingImpl(psi, dofsPerNode,smGlobalIndices,comm)

  # write weighting matrix only if it has data in it
  if isinstance(weighting, np.ndarray):
    _writeResultsToFile(dic["outDir"], weighting, dic["isBinary"], comm)

#---------------------------------------------------------------------
# this is the entry function visibile outside, all the other functions
# above are implementation details (note the _ prepending all names)
# and should not be exposed outside
#---------------------------------------------------------------------
def computeLspgWeighting(**args):
  calledFromDriver = True if 'fromDriver' in args.keys() else False

  # if this is called from driverscript
  if calledFromDriver:
    assert(len(args) == 3)
    assert('communicator' in args.keys())
    assert('yamldic'      in args.keys())
    _computeLspgWeightingReadYaml(args['communicator'],
                                  args['yamldic'])

  # if not called from driver, it is called from within python
  elif not calledFromDriver:
    assert('residualBasis'      in args.keys())
    assert('sampleMeshIndices'  in args.keys())
    assert('dofsPerMeshNode'    in args.keys())
    comm = None if 'communicator'  not in args.keys() else args['communicator']

    return _computeLspgWeightingImpl(args["residualBasis"],
                                     args["dofsPerMeshNode"],
                                     args['sampleMeshIndices'],
                                     comm)

max_stars_repo_name: Pressio/pressio-hyperreduction

max_issues_repo_name: Pressio/pressio-hyperreduction

max_forks_repo_name: Pressio/pressio-tools