text:
#!/usr/bin/env python3
import os

from parsl import python_app

def get_logger(path, filename='identifier.log'):
    import logging
    import os

    logfile = os.path.join(path, filename)
    formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")
    file_handler = logging.FileHandler(logfile)
    file_handler.setFormatter(formatter)
    log = logging.getLogger('identifier')
    log.setLevel(logging.DEBUG)
    log.addHandler(file_handler)

    return log

def read_inputs(path, filename='job.json'):
    import os
    import json
    with open(os.path.join(path, filename)) as json_file:
        data = json.load(json_file)

    return data

# # def load_heartbeat():
# #     import json
# #     with open('heartbeat.json') as json_file:
# #         data = json.load(json_file)
# #         return data

def write_job_file(path, data):
    import json
    import os
    with open(os.path.join(path, 'job.json'), 'w') as json_file:
        json.dump(data, json_file)

# @python_app(executors=["htcondor_1"])
@python_app()
def retrieve_asteroids(type, values):

    from dao import AsteroidDao

    asteroids = None
    if type == 'name':
        asteroids = AsteroidDao().get_asteroids_by_names(
            names=values.split(';')
        )
    else:
        asteroids = AsteroidDao().get_asteroids_by_dynclass(
            dynclass=values
        )

    for asteroid in asteroids:
        asteroid.update({
            'status': 'running',
            'ccds': list()
        })

    return asteroids

def date_to_jd(date_obs, exptime, leap_second):
    """Aplica uma correção a data de observação e converte para data juliana

    Correção para os CCDs do DES:
        date = date_obs + 0.5 * (exptime + 1.05)

    Args:
        date_obs (datetime): Data de observação do CCD "date_obs"
        exptime (float): Tempo de exposição do CCD "exptime"
        lead_second (str): Path para o arquivo leap second a ser utilizado por exemplo: '/archive/lead_second/naif0012.tls'

    Returns:
        str: Data de observação corrigida e convertida para julian date.
    """
    import spiceypy as spice

    # Calcula a correção
    correction = (exptime + 1.05)/2

    # Carrega o lead second na lib spicepy
    spice.furnsh(leap_second)

    # Converte a date time para JD
    date_et = spice.utc2et(str(date_obs).split('+')[0] + " UTC")
    date_jdutc = spice.et2utc(date_et, 'J', 14)

    # Remove a string JD retornada pela lib
    midtime = date_jdutc.replace('JD ', '')

    # Soma a correção
    jd = float(midtime) + correction/86400

    spice.kclear()

    return jd

def get_bsp_from_jpl(identifier, initial_date, final_date, email, directory):
    """Download bsp files from JPL database

        Bsp files, which have information to generate
        the ephemeris of the objects, will be downloaded
        The files will be named as (without spaces): [identifier].bsp

        Important:
            it is able to download bsp files of neither planets nor satellites

    Args:
        identifier (str): Identifier of the object.
            It can be the name, number or SPK ID.
            It can also be a list of objects.
            Examples:
                '2137295'
                '1999 RB216'
                '137295'
                'Chariklo'
        initial_date (str): Date the bsp file is to begin, within span [1900-2100].
            Examples:
                '2003-02-01'
                '2003-3-5'
        final_date (str): Date the bsp file is to end, within span [1900-2100].
            Must be more than 32 days later than [initial_date].
            Examples:
                '2006-01-12'
                '2006-1-12'
        email (str): User's e-mail contact address.
            Example: username@user.domain.name
        directory (str): Directory path to save the bsp files.

    Returns:
        Path: file path for .bsp file.
    """
    import pathlib
    import shutil
    from datetime import datetime as dt

    import requests
    import spiceypy as spice

    date1 = dt.strptime(initial_date, '%Y-%m-%d')
    date2 = dt.strptime(final_date, '%Y-%m-%d')
    diff = date2 - date1

    if diff.days <= 32:
        raise ValueError(
            'The [final_date] must be more than 32 days later than [initial_date]')

    urlJPL = 'https://ssd.jpl.nasa.gov/x/smb_spk.cgi?OPTION=Make+SPK'

    path = pathlib.Path(directory)
    if not path.exists():
        raise ValueError('The directory {} does not exist!'.format(path))

    filename = identifier.replace(' ', '') + '.bsp'

    parameters = {
        'OBJECT': identifier,
        'START': date1.strftime('%Y-%b-%d'),
        'STOP': date2.strftime('%Y-%b-%d'),
        'EMAIL': email,
        'TYPE': '-B'
    }

    r = requests.get(urlJPL, params=parameters, stream=True)
    bspFormat = r.headers['Content-Type']
    if r.status_code == requests.codes.ok and bspFormat == 'application/download':
        file_path = path.joinpath(filename)
        with open(file_path, 'wb') as f:
            r.raw.decode_content = True
            shutil.copyfileobj(r.raw, f)

        return file_path
    else:
        # TODO: Add a Debug
        raise Exception(
            "It was not able to download the bsp file for object.")

def findSPKID(bsp):
    """Search the spk id of a small Solar System object from bsp file

    Args:
        bsp (str): File path for bsp jpl file.

    Returns:
        str: Spk id of Object
    """
    import spiceypy as spice

    bsp = [bsp]
    spice.furnsh(bsp)

    i = 0
    kind = 'spk'
    fillen = 256
    typlen = 33
    srclen = 256
    keys = ['Target SPK ID   :', 'ASTEROID_SPK_ID =']
    n = len(keys[0])

    name, kind, source, loc = spice.kdata(i, kind, fillen, typlen, srclen)
    flag = False
    spk = ''
    while not flag:
        try:
            m, header, flag = spice.dafec(loc, 1)
            row = header[0]
            if row[:n] in keys:
                spk = row[n:].strip()
                break
        except:
            break
    return spk

def geo_topo_vector(longitude, latitude, elevation, jd):
    '''
    Transformation from [longitude, latitude, elevation] to [x,y,z]
    '''
    from astropy.coordinates import GCRS, EarthLocation
    from astropy.time import Time
    import numpy as np

    loc = EarthLocation(longitude, latitude, elevation)

    time = Time(jd, scale='utc', format='jd')
    itrs = loc.get_itrs(obstime=time)
    gcrs = itrs.transform_to(GCRS(obstime=time))

    r = gcrs.cartesian

    # convert from m to km
    x = r.x.value/1000.0
    y = r.y.value/1000.0
    z = r.z.value/1000.0

    return np.array([x, y, z])

def compute_theoretical_positions(spkid, ccds, bsp_jpl, bsp_planetary, leap_second, location):

    import spiceypy as spice
    import numpy as np
    from library import geo_topo_vector

    # TODO: Provavelmente esta etapa é que causa a lentidão desta operação
    # Por que carrega o arquivo de ephemeris planetarias que é pesado
    # Load the asteroid and planetary ephemeris and the leap second (in order)
    spice.furnsh(bsp_planetary)
    spice.furnsh(leap_second)
    spice.furnsh(bsp_jpl)

    results = list()

    for ccd in ccds:
        date_jd = ccd['date_jd']

        # Convert dates from JD to et format. "JD" is added due to spice requirement
        date_et = spice.utc2et(str(date_jd) + " JD UTC")

        # Compute geocentric positions (x,y,z) in km for each date with light time correction
        r_geo, lt_ast = spice.spkpos(spkid, date_et, 'J2000', 'LT', '399')

        lon, lat, ele = location
        l_ra, l_dec = [], []

        # Convert from geocentric to topocentric coordinates
        r_topo = r_geo - geo_topo_vector(lon, lat, ele, float(date_jd))

        # Convert rectangular coordinates (x,y,z) to range, right ascension, and declination.
        d, rarad, decrad = spice.recrad(r_topo)

        # Transform RA and Decl. from radians to degrees.
        ra = np.degrees(rarad)
        dec = np.degrees(decrad)

        ccd.update({
            'date_et': date_et,
            'geocentric_positions': list(r_geo),
            'topocentric_positions': list(r_topo),
            'theoretical_coordinates': [ra, dec]
        })

        results.append(ccd)

    spice.kclear()

    return results

# @python_app(executors=["htcondor_1"])
@python_app()
def retrieve_ccds_by_asteroid(asteroid, leap_second):
    from dao import AsteroidDao
    from library import date_to_jd

    ccds = AsteroidDao().ccds_by_asteroid(
        asteroid_name=asteroid['name']
    )

    for ccd in ccds:
        ccd.update({
            'date_obs': str(ccd['date_obs']),
            'date_jd': date_to_jd(ccd['date_obs'], ccd['exptime'], leap_second),
        })

    asteroid['ccds'] = ccds

    return asteroid

# @python_app(executors=["htcondor_1"])
@python_app()
def retrieve_bsp_by_asteroid(name, initial_date, final_date, job_path):

    import os
    import pathlib
    import traceback
    from library import get_bsp_from_jpl
    from config import JPL_EMAIL

    bsp = dict({
        'status': None,
        'filename': None,
        'path': None,
        'file_size': None,
    })

    try:
        # Criar o diretório para o asteroid.
        asteroid_path = pathlib.Path.joinpath(
            pathlib.Path(job_path), name.replace(' ', '_'))
        pathlib.Path(asteroid_path).mkdir(parents=True, exist_ok=True)

        if not asteroid_path.exists():

            # Download BSP from JPL
            bsp_path = get_bsp_from_jpl(
                name, initial_date, final_date, JPL_EMAIL, asteroid_path.absolute()
            )

        else:
            # Já existe diretório verifica se existe o arquivo
            a_path = asteroid_path.joinpath('%s.bsp' % name.replace(' ', '_'))

            if not a_path.exists():
                # Download BSP from JPL
                bsp_path = get_bsp_from_jpl(
                    name, initial_date, final_date, JPL_EMAIL, asteroid_path.absolute()
                )
            else:
                # Arquivo já existe faz nada
                bsp_path = a_path

        bsp.update({
            'status': 'success',
            'filename': os.path.basename(bsp_path),
            'path': str(bsp_path),
            'file_size': os.path.getsize(bsp_path)
        })

    except Exception as e:
        trace = traceback.format_exc()

        msg = "Failed on retrive asteroids BSP from JPL. %s" % e

        bsp['status'] = 'failure'
        bsp['error'] = msg
        bsp['traceback'] = trace

    finally:
        return bsp

# @python_app(executors=["htcondor"])
@python_app()
def theoretical_positions(asteroid, bsp_planetary, leap_second, observatory_location):

    from library import compute_theoretical_positions
    import traceback

    try:
        # Recuperar o SPK Id do objeto a partir do seu BSP
        asteroid['spkid'] = findSPKID(
            asteroid['bsp_jpl']['path'])

        ccds = compute_theoretical_positions(
            asteroid['spkid'],
            asteroid['ccds'],
            asteroid['bsp_jpl']['path'],
            bsp_planetary,
            leap_second,
            observatory_location
        )

        asteroid['ccds'] = ccds

    except Exception as e:
        trace = traceback.format_exc()

        msg = "Failed on retrive asteroids BSP from JPL. %s" % e

        asteroid['status'] = 'failure'
        asteroid['error'] = msg
        asteroid['traceback'] = trace

    finally:
        return asteroid

# @python_app(executors=["htcondor"])
@python_app()
def observed_positions(idx, name, asteroid_id, ccd, asteroid_path, radius=2):

    import os
    import numpy as np
    from astropy.io import fits
    from library import get_logger

    import numpy as np
    from astropy import units as u
    from astropy.coordinates import SkyCoord

    obs_coordinates = None

    log = get_logger(asteroid_path, '%s_obs_%s.log' %
                     (name.replace(' ', '_'), idx))

    log.info("Calculating observed positions")
    log.debug("Asteroid: %s" % name)
    log.debug("IDX: %s" % idx)
    log.debug("CCD: %s" % ccd['path'])

    try:
        # Coordenadas teroricas
        ra_jpl, dec_jpl = ccd['theoretical_coordinates']

        file_path = os.path.join(ccd['path'], ccd['filename'])

        # Le o catalogo Fits
        hdul = fits.open(file_path)
        rows = hdul[2].data

        a_ra_des = rows['ALPHA_J2000']
        a_dec_des = rows['DELTA_J2000']
        a_magpsf = rows['MAG_PSF']
        a_magerrpsf = rows['MAGERR_PSF']

        catJPL = SkyCoord(ra=[ra_jpl]*u.deg, dec=[dec_jpl]*u.deg, frame='icrs')
        catD00 = SkyCoord(ra=a_ra_des*u.deg, dec=a_dec_des*u.deg, frame='icrs')
        id1, d2d1, d3d = catD00.match_to_catalog_sky(catJPL)

        # search the index and value of element with minimal distance
        IDX_D00 = np.argmin(d2d1.arcsec)
        dmin = d2d1[IDX_D00].arcsec

        ra, dec, match_idx = None, None, None
        if dmin < radius:
            ra = a_ra_des[IDX_D00]
            dec = a_dec_des[IDX_D00]
            match_idx = IDX_D00

        if ra is not None and dec is not None:
            # Calcular "observed minus calculated"
            observed_omc = [float((ra-ra_jpl)*3600*np.cos(np.radians(dec_jpl))),
                            float((dec-dec_jpl)*3600)]
            ccd.update({
                'observed_coordinates': [float(ra), float(dec)],
                'observed_omc': observed_omc,
                'magpsf': float(a_magpsf[match_idx]),
                'magerrpsf': float(a_magerrpsf[match_idx])
            })

            # Cria um objeto com os dados da posição observada,
            # Esta etapa está aqui para aproveitar o paralelismo e adiantar a consolidação.
            obs_coordinates = dict({
                'name': name,
                'asteroid_id': asteroid_id,
                'ccd_id': ccd['id'],
                'date_obs': ccd['date_obs'],
                'date_jd': ccd['date_jd'],
                'ra': ccd['observed_coordinates'][0],
                'dec': ccd['observed_coordinates'][1],
                'offset_ra': ccd['observed_omc'][0],
                'offset_dec': ccd['observed_omc'][1],
                'mag_psf': ccd['magpsf'],
                'mag_psf_err': ccd['magerrpsf'],
            })

            log.info("Observed Coordinates: %s" % ccd['observed_coordinates'])

        else:
            ccd.update({
                'observed_coordinates': None,
                'info': 'Could not find a position.'
            })

            log.info("Could not find a position.")

    except Exception as e:
        ccd.update({
            'observed_coordinates': None,
            'error': 'Failed in match position stage with exception: %s' % str(e)
        })
        log.error('Failed in match position stage with exception: %s' % str(e))

    finally:
        log.info('Calculating observed positions Finish')
        return (name, ccd, obs_coordinates)

# @python_app(executors=["htcondor_1"])
@python_app()
def ingest_observations(observations):

    import pandas as pd
    from io import StringIO
    from dao import AsteroidDao, ObservationDao

    # Apaga as observations já registradas para este asteroid antes de inserir.
    name = observations[0]['name']
    ObservationDao().delete_by_asteroid_name(name)

    df_obs = pd.DataFrame(observations, columns=[
        'name', 'date_obs', 'date_jd', 'ra', 'dec', 'offset_ra',
        'offset_dec', 'mag_psf', 'mag_psf_err', 'asteroid_id', 'ccd_id'])

    # TODO: Tratar os dados se necessário
    data = StringIO()
    df_obs.to_csv(
        data,
        sep="|",
        header=True,
        index=False,
    )
    data.seek(0)

    tablename = 'des_observation'

    # Sql Copy com todas as colunas que vão ser importadas e o formato do csv.
    sql = "COPY %s (name, date_obs, date_jd, ra, dec, offset_ra, offset_dec, mag_psf, mag_psf_err, asteroid_id, ccd_id) FROM STDIN with (FORMAT CSV, DELIMITER '|', HEADER);" % tablename

    rowcount = AsteroidDao().import_with_copy_expert(sql, data)

    return rowcount

# @python_app(executors=["htcondor"])
@python_app()
def write_asteroid_data(asteroid):
    import json
    import gzip
    import os

    filepath = os.path.join(
        asteroid['path'], '%s.json.gz' % asteroid['name'].replace(' ', '_'))

    with gzip.open(filepath, 'wt', encoding='UTF-8') as zipfile:
        json.dump(asteroid, zipfile)

    return asteroid

# @python_app(executors=["htcondor"])
@python_app()
def update_job(job):
    from dao import AstrometryJobDao

    AstrometryJobDao().update_job(job)

max_stars_repo_name: linea-it/tno

max_issues_repo_name: linea-it/tno

max_forks_repo_name: linea-it/tno

reason: import numpy,from astropy


text:
import numpy.random as random
import json
import os

from rooms import Room
from things import Thing
from custom_json import custom_load

class PlanetRoomGenerator:
    """

    """
    def __init__(self, seed, limit=1):
        self.seed = seed
        with open(os.path.join("Generators", (self.__class__.__name__.lower() + '.json')), 'r') as file:
            self.generator_data = json.load(file)
        with open(os.path.join("Generators", "threats.json"), 'r') as file:
            self.threat_data = json.load(file)
        self.limit = limit

    def generate_landing_spot(self, game, solarsystem, planet):
        i = 0
        for i, p_key in enumerate(solarsystem.contents.keys()):
            p = solarsystem.contents[p_key]['obj']
            if p == planet:
                break
        seed = self.seed + solarsystem.name_seed + i
        game.rooms = dict()
        game.rooms['t0p0'] = self.__generate_room('t0p0', seed, planet, solarsystem)
        self.generate_rooms(game, solarsystem, planet)
        game.refresh_things()

    def generate_rooms(self, game, solarsystem, planet):
        i = 0
        for i, p_key in enumerate(solarsystem.contents.keys()):
            p = solarsystem.contents[p_key]['obj']
            if p == planet:
                break
        seed = self.seed + solarsystem.name_seed + i
        current_room = game.rooms[game.game_state['current room']]
        current_chapter = game.chapters[game.game_state['current chapter']]
        rooms_dir, new_rooms, old_rooms = self._adj_room_coordinates(current_room, game.rooms)

        current_room.directions = rooms_dir
        for new_room_key in new_rooms:
            new_room = self.__generate_room(new_room_key, seed, planet, solarsystem)
            game.rooms[new_room_key] = new_room
            current_chapter.map_files.append(new_room_key)

        for old_room_key in old_rooms:
            if old_room_key == game.game_state['current room']:
                continue
            del game.rooms[old_room_key]
            try:
                current_chapter.map_files.remove(old_room_key)
            except ValueError:
                pass
        game.refresh_things()

    def __generate_room(self, room_key, seed, planet, solarsystem):
        t = int(room_key[1:self.limit+1])
        p = int(room_key[2+self.limit:])
        seed *= (t+1)
        seed /= (p+1)
        seed += ((t+1)*(p+1))
        seed = int(seed)
        random.seed(seed)

        room_types = []
        for elem in planet.entity_state['features']['geology']:
            room_types.append(elem)
        if planet.lifeforms in {"Animal Life", "Advanced Life"}:
            room_types.append("forest")
        if planet.lifeforms == "Former Advanced Life":
            room_types.append("abandoned city")
        if planet.lifeforms == "Advanced Life":
            room_types.append("city")

        new_room = Room(key=room_key, display_name=room_key, reference_noun=room_key)
        new_room.entity_state['room type'] = random.choice(room_types)
        new_room.display_name = new_room.entity_state['room type']
        self.generate_description(new_room, seed, planet, solarsystem)
        self.__load_threat_objects(planet, new_room)
        colony = Thing(key='colony', reference_noun='colony', display_name='colony',
                       description='A Union Colony has not yet been set up on this planet.',
                       as_dirobj={'Setup': True},
                       action_description={"Setup":
                                           "Thus, you begin the construction of your colony on planet {planet}."})
        new_room += colony
        return new_room

    def generate_description(self, room, seed, planet, solarsystem):
        room_type = room.entity_state['room type']

        room_key = room.key
        t = int(room_key[1:1 + self.limit])
        p = int(room_key[2 + self.limit:])
        description = f"Latitude: {p}, Longitude: {t}.\n"

        random.seed(seed)
        description += self.generator_data[room_type]['description']
        template = random.choice(self.generator_data[room_type]['templates'])

        clouds = ''
        rings = ''
        atmosphere = ''
        radiation = ''
        temperature = ''

        if planet.atmosphere_type == "Cloudy Atmosphere":
            clouds = 'The sky is barely visible through the clouds covering the entire planet.\n'
        if planet.entity_state['features']['rings'] == "extensive":
            rings = f"The view of {planet.display_name}'s rings is breathtaking.\n"
        if planet.atmosphere_type == "No Atmosphere":
            atmosphere = "Your pressure sensors record almost no atmosphere. " \
                         "You wouldn't be able to breath without a suit on."
        if planet.atmosphere_type == "Thick Atmosphere":
            atmosphere = 'There is no need for a suit here, maybe just some oxygen tanks for extra intake.'
        if planet.atmosphere_type == "Cloudy Atmosphere":
            atmosphere = 'There is no need for a suit here, maybe just some oxygen tanks for extra intake.'
        if planet.atmosphere_type == "Acid Atmosphere":
            atmosphere = 'Something corrosive in the air is slowly eating away on your equipment.'
        if solarsystem.habitable:
            radiation = 'Radiation levels are tolerable.'
        if not solarsystem.habitable:
            radiation = "Radiation levels are really high, it's almost too dangerous to be walking on the surface."
        if planet.temperature == "Warm":
            temperature = 'The temperature is hot enough to make you uncomfortable, ' \
                          'but you reckon you can withstand it.'
        if planet.temperature == "Temperate":
            temperature = "The temperature is close enough to a mild spring day on Earth."
        if planet.temperature == "Cold":
            temperature = "The temperature hovers just above zero, but you're used to the cold."
        if planet.temperature == "Frozen":
            temperature = "The temperature is constantly below zero. Nothing but extremophiles could survive here."
        if planet.temperature == "Icy":
            temperature = "Even with the cutting edge anti-freeze tech of your suit, the cold is unbearable.\n" \
                          "There are small ponds of liquid methane around you."

        template = template.format(num_stars=len(solarsystem.star_names),
                                   star_plural='' if len(solarsystem.star_names) == 1 else 's',
                                   clouds=clouds, sky_color=planet.entity_state['features']['sky color'],
                                   num_satellites=planet.num_satellites,
                                   satellite_plural='' if planet.num_satellites == 1 else 's',
                                   rings=rings, atmosphere=atmosphere, radiation=radiation, temperature=temperature)

        room.examine_description = description + template
        room.description = description
        room.audible_description = self.generator_data[room_type]['audible']

    def __load_threat_objects(self, planet, room):
        clues = custom_load(os.path.join("Generators", "clues.json"))
        threats = planet.entity_state['features']['threats']
        for threat in threats.keys():
            if threats[threat]:
                # Place clue
                if self.threat_data[threat]['clues']:
                    place_clue = random.choice([True, False], p=[0.8, 0.2])
                    if place_clue:
                        clue_key = random.choice(self.threat_data[threat]['clues'])
                        clue = clues[clue_key]
                        room += clue

    def generate_things(self):
        pass

    def _adj_room_coordinates(self, room, rooms_dict):
        room_key = room.key
        t_coord = int(room_key[1:1+self.limit])
        p_coord = int(room_key[2+self.limit:])

        directions = dict()
        adj_rooms = []

        t, p = self._coordinates(t_coord, p_coord, 0, 1)
        new_room_key = f"t{t}p{p}"
        adj_rooms.append(new_room_key)
        directions['North'] = {"room": new_room_key, "desc": f"Latitude: {p}, Longitude: {t}."}

        t, p = self._coordinates(t_coord, p_coord, -1, 1)
        new_room_key = f"t{t}p{p}"
        adj_rooms.append(new_room_key)
        directions['Northeast'] = {"room": new_room_key, "desc": f"Latitude: {p}, Longitude: {t}."}

        t, p = self._coordinates(t_coord, p_coord, 1, 1)
        new_room_key = f"t{t}p{p}"
        adj_rooms.append(new_room_key)
        directions['Northwest'] = {"room": new_room_key, "desc": f"Latitude: {p}, Longitude: {t}."}

        t, p = self._coordinates(t_coord, p_coord, -1, 0)
        new_room_key = f"t{t}p{p}"
        adj_rooms.append(new_room_key)
        directions['East'] = {"room": new_room_key, "desc": f"Latitude: {p}, Longitude: {t}."}

        t, p = self._coordinates(t_coord, p_coord, 1, 0)
        new_room_key = f"t{t}p{p}"
        adj_rooms.append(new_room_key)
        directions['West'] = {"room": new_room_key, "desc": f"Latitude: {p}, Longitude: {t}."}

        t, p = self._coordinates(t_coord, p_coord, 0, -1)
        new_room_key = f"t{t}p{p}"
        adj_rooms.append(new_room_key)
        directions['South'] = {"room": new_room_key, "desc": f"Latitude: {p}, Longitude: {t}."}

        t, p = self._coordinates(t_coord, p_coord, -1, -1)
        new_room_key = f"t{t}p{p}"
        adj_rooms.append(new_room_key)
        directions['Southeast'] = {"room": new_room_key, "desc": f"Latitude: {p}, Longitude: {t}."}

        t, p = self._coordinates(t_coord, p_coord, 1, -1)
        new_room_key = f"t{t}p{p}"
        adj_rooms.append(new_room_key)
        directions['Southwest'] = {"room": new_room_key, "desc": f"Latitude: {p}, Longitude: {t}."}

        new_rooms = []
        old_rooms = []
        for room_key in rooms_dict.keys():
            if room_key not in adj_rooms:
                old_rooms.append(room_key)

        for room_key in adj_rooms:
            if room_key not in rooms_dict.keys():
                new_rooms.append(room_key)

        return directions, new_rooms, old_rooms

    def _coordinates(self, theta, phi, dtheta, dphi):
        new_theta = theta + dtheta
        new_phi = phi + dphi

        if new_phi < 0:
            new_phi += 10**self.limit

        if new_theta < 0:
            new_theta += 10**self.limit

        new_theta = new_theta % (10**self.limit)
        new_phi = new_phi % (10**self.limit)

        return new_theta, new_phi

ext: py

max_stars_repo_name: Frankkie/Thesis-Project-IF-Game

max_issues_repo_name: Frankkie/Thesis-Project-IF-Game

max_forks_repo_name: Frankkie/Thesis-Project-IF-Game

reason: import numpy


text:
import numpy as np
import cv2 as cv

img = cv.imread('1.jpeg',cv.IMREAD_COLOR)

#we have opencv fonts so TRIPLEX is one of the fonts of opencv
font = cv.FONT_HERSHEY_TRIPLEX

#can add text by putText function or method where parameters are 1. where to write(image), 2.text , 3.starting point, 4.which type of font,
#5.size of font, 6.color of font, 7.thickness of font, 8.recommanded by openCV to have LINE_AA when adding a text on image.
cv.putText(img, 'I can write here', (0,120), font, 1,(20,21,253), 3, cv.LINE_AA)

cv.imshow('image',img)
cv.waitKey(0)
cv.destroyAllWindows()


text:
##############################################################################
#
# Copyright (c) 2003-2020 by The University of Queensland
# http://www.uq.edu.au
#
# Primary Business: Queensland, Australia
# Licensed under the Apache License, version 2.0
# http://www.apache.org/licenses/LICENSE-2.0
#
# Development until 2012 by Earth Systems Science Computational Center (ESSCC)
# Development 2012-2013 by School of Earth Sciences
# Development from 2014 by Centre for Geoscience Computing (GeoComp)
# Development from 2019 by School of Earth and Environmental Sciences
#
##############################################################################

"""Cost functions for inversions with one or more forward models"""

from __future__ import print_function, division

__copyright__="""Copyright (c) 2003-2020 by The University of Queensland
http://www.uq.edu.au
Primary Business: Queensland, Australia"""
__license__="""Licensed under the Apache License, version 2.0
http://www.apache.org/licenses/LICENSE-2.0"""
__url__="https://launchpad.net/escript-finley"

__all__ = [ 'InversionCostFunction']

from .costfunctions import MeteredCostFunction
from .mappings import Mapping
from .forwardmodels import ForwardModel
from esys.escript import Data, inner, interpolate
import numpy as np

class InversionCostFunction(MeteredCostFunction):
    """
    Class to define cost function *J(m)* for inversion with one or more
    forward models based on a multi-valued level set function *m*:

    *J(m) = J_reg(m) + sum_f mu_f * J_f(p)*

    where *J_reg(m)* is the regularization and cross gradient component of the
    cost function applied to a level set function *m*, *J_f(p)* are the data
    defect cost functions involving a physical forward model using the
    physical parameter(s) *p* and *mu_f* is the trade-off factor for model f.

    A forward model depends on a set of physical parameters *p* which are
    constructed from components of the level set function *m* via mappings.

    Example 1 (single forward model):
         m=Mapping()
         f=ForwardModel()
         J=InversionCostFunction(Regularization(), m, f)

    Example 2 (two forward models on a single valued level set)
         m0=Mapping()
         m1=Mapping()
         f0=ForwardModel()
         f1=ForwardModel()

         J=InversionCostFunction(Regularization(), mappings=[m0, m1], forward_models=[(f0, 0), (f1,1)])

    Example 3 (two forward models on 2-valued level set)
         m0=Mapping()
         m1=Mapping()
         f0=ForwardModel()
         f1=ForwardModel()

         J=InversionCostFunction(Regularization(self.numLevelSets=2), mappings=[(m0,0), (m1,0)], forward_models=[(f0, 0), (f1,1)])

    :note: If provides_inverse_Hessian_approximation is true, then the class
          provides an approximative inverse of the Hessian operator.
    """
    provides_inverse_Hessian_approximation=True

    def __init__(self, regularization, mappings, forward_models):
        """
        constructor for the cost function.
        Stores the supplied object references and sets default weights.

        :param regularization: the regularization part of the cost function
        :type regularization: `Regularization`
        :param mappings: the mappings to calculate physical parameters from the
                         regularization. This is a list of 2-tuples *(map, i)*
                         where the first component map defines a `Mapping` and
                         the second component *i* defines the index of the
                         component of level set function to be used to
                         calculate the mapping. Items in the list may also be
                         just `Mapping` objects in which case the entire level
                         set function is fed into the `Mapping` (typically used
                         for a single-component level set function.
        :type mappings: `Mapping` or ``list``
        :param forward_models: the forward models involved in the calculation
                               of the cost function. This is a list of 2-tuples
                               *(f, ii)* where the first component f defines a
                               `ForwardModel` and the second component *ii* a
                               list of indexes referring to the physical
                               parameters in the `mappings` list. The 2-tuple
                               can be replaced by a `ForwardModel` if the
                               `mappings` list has a single entry.
        :param forward_models: `ForwardModel` or ``list``
        """
        super(InversionCostFunction, self).__init__()
        self.regularization=regularization
        self.numLevelSets = self.regularization.getNumLevelSets()

        if isinstance(mappings, Mapping):
            mappings = [ mappings ]

        self.mappings=[]
        for i in range(len(mappings)):
            mm=mappings[i]
            if isinstance(mm, Mapping):
                m=mm
                if self.numLevelSets>1:
                    idx=[ p for p in range(self.numLevelSets)]
                else:
                    idx=None
            elif len(mm) == 1:
                m=mm[0]
                if self.numLevelSets>1:
                    idx=[ p for p in range(self.numLevelSets)]
                else:
                    idx=None
            else:
                m=mm[0]
                if isinstance(mm[1], int):
                    idx=[mm[1]]
                else:
                    idx=list(mm[1])
                if self.numLevelSets>1:
                    for k in idx:
                        if  k < 0  or k > self.numLevelSets-1:
                            raise ValueError("level set index %s is out of range."%(k,))

                else:
                    if idx[0] != 0:
                        raise ValueError("Level set index %s is out of range."%(idx[0],))
                    else:
                        idx=None
            self.mappings.append((m,idx))
        self.numMappings=len(self.mappings)

        if isinstance(forward_models, ForwardModel):
            forward_models = [ forward_models ]
        self.forward_models=[]
        for i in range(len(forward_models)):
            f=forward_models[i]
            if isinstance(f, ForwardModel):
                idx=[0]
                fm=f
            elif len(f) == 1:
                idx=[0]
                fm=f[0]
            else:
                if isinstance(f[1],int):
                    idx=[f[1]]
                else:
                    idx=list(f[1])
                for k in idx:
                    if k<0 or k> self.numMappings:
                        raise ValueError("mapping index %s in model %s is out of range."%(k,i))
                fm=f[0]
            self.forward_models.append((fm,idx))
        self.numModels=len(self.forward_models)

        trafo = self.regularization.getCoordinateTransformation()
        for m in self.forward_models:
            if not m[0].getCoordinateTransformation() == trafo:
                raise ValueError("Coordinate transformation for regularization and model don't match.")

        self.__num_tradeoff_factors = self.regularization.getNumTradeOffFactors() + self.numModels
        self.setTradeOffFactorsModels()

    def getDomain(self):
        """
        returns the domain of the cost function

        :rtype: `Domain`
        """
        self.regularization.getDomain()

    def getNumTradeOffFactors(self):
        """
        returns the number of trade-off factors being used including the
        trade-off factors used in the regularization component.

        :rtype: ``int``
        """
        return self.__num_tradeoff_factors

    def getForwardModel(self, idx=None):
        """
        returns the *idx*-th forward model.

        :param idx: model index. If cost function contains one model only `idx`
                    can be omitted.
        :type idx: ``int``
        """
        if idx==None: idx=0
        return self.forward_models[idx][0]

    def getRegularization(self):
        """
        returns the regularization

        :rtype: `Regularization`
        """
        return self.regularization

    def setTradeOffFactorsModels(self, mu=None):
        """
        sets the trade-off factors for the forward model components.

        :param mu: list of the trade-off factors. If not present ones are used.
        :type mu: ``float`` in case of a single model or a ``list`` of
                  ``float`` with the length of the number of models.
        """
        if mu==None:
            self.mu_model=np.ones((self.numModels, ))
        else:
            if self.numModels > 1:
                mu=np.asarray(mu, dtype=float)
                if min(mu) > 0:
                    self.mu_model= mu
                else:
                    raise ValueError("All values for trade-off factor mu must be positive.")
            else:
                mu=float(mu)
                if mu > 0:
                    self.mu_model= [mu, ]
                else:
                    raise ValueError("Trade-off factor must be positive.")

    def getTradeOffFactorsModels(self):
        """
        returns the trade-off factors for the forward models

        :rtype: ``float`` or ``list`` of ``float``
        """
        if self.numModels>1:
            return self.mu_model
        else:
            return self.mu_model[0]

    def setTradeOffFactorsRegularization(self, mu=None, mu_c=None):
        """
        sets the trade-off factors for the regularization component of the
        cost function, see `Regularization` for details.

        :param mu: trade-off factors for the level-set variation part
        :param mu_c: trade-off factors for the cross gradient variation part
        """
        self.regularization.setTradeOffFactorsForVariation(mu)
        self.regularization.setTradeOffFactorsForCrossGradient(mu_c)

    def setTradeOffFactors(self, mu=None):
        """
        sets the trade-off factors for the forward model and regularization
        terms.

        :param mu: list of trade-off factors.
        :type mu: ``list`` of ``float``
        """
        if mu is None:
            mu=np.ones((self.__num_tradeoff_factors,))
        self.setTradeOffFactorsModels(mu[:self.numModels])
        self.regularization.setTradeOffFactors(mu[self.numModels:])

    def getTradeOffFactors(self, mu=None):
        """
        returns a list of the trade-off factors.

        :rtype: ``list`` of ``float``
        """
        mu1=self.getTradeOffFactorsModels(mu[:self.numModels])
        mu2=self.regularization.getTradeOffFactors()
        return [ m for m in mu1] + [ m for m in mu2]

    def createLevelSetFunction(self, *props):
        """
        returns an instance of an object used to represent a level set function
        initialized with zeros. Components can be overwritten by physical
        properties `props`. If present entries must correspond to the
        `mappings` arguments in the constructor. Use ``None`` for properties
        for which no value is given.
        """
        m=self.regularization.getPDE().createSolution()
        if len(props) > 0:
            for i in range(self.numMappings):
                if props[i]:
                    mp, idx=self.mappings[i]
                    m2=mp.getInverse(props[i])
                    if idx:
                        if len(idx) == 1:
                            m[idx[0]]=m2
                        else:
                            for k in range(idx): m[idx[k]]=m2[k]
                    else:
                        if isinstance(m2, Data):
                           m=interpolate(m2, m.getFunctionSpace())
                        else:
                           m=Data(m2, m.getFunctionSpace())
        return m

    def getProperties(self, m, return_list=False):
        """
        returns a list of the physical properties from a given level set
        function *m* using the mappings of the cost function.

        :param m: level set function
        :type m: `Data`
        :param return_list: if ``True`` a list is returned.
        :type return_list: ``bool``
        :rtype: ``list`` of `Data`
        """

        props=[]
        for i in range(self.numMappings):
            mp, idx=self.mappings[i]
            if idx:
                if len(idx)==1:
                    p=mp.getValue(m[idx[0]])
                else:
                    m2=Data(0.,(len(idx),),m.getFunctionSpace())
                    for k in range(len(idx)): m2[k]=m[idx[k]]
                    p=mp.getValue(m2)
            else:
                p=mp.getValue(m)
            props.append(p)
        if self.numMappings > 1 or return_list:
            return props
        else:
            return props[0]

    def _getDualProduct(self, x, r):
        """
        Returns the dual product, see `Regularization.getDualProduct`

        :type x: `Data`
        :type r: `ArithmeticTuple`
        :rtype: ``float``
        """
        return self.regularization.getDualProduct(x, r)

    def _getArguments(self, m):
        """
        returns pre-computed values that are shared in the calculation of
        *J(m)* and *grad J(m)*. In this implementation returns a tuple with the
        mapped value of ``m``, the arguments from the forward model and the
        arguments from the regularization.

        :param m: current approximation of the level set function
        :type m: `Data`
        :return: tuple of of values of the parameters, pre-computed values
                 for the forward model and pre-computed values for the
                 regularization
        :rtype: ``tuple``
        """
        args_reg=self.regularization.getArguments(m)
        # cache for physical parameters:
        props=self.getProperties(m, return_list=True)
        args_f=[]
        for i in range(self.numModels):
            f, idx=self.forward_models[i]
            pp=tuple( [ props[k] for k in idx] )
            aa=f.getArguments(*pp)
            args_f.append(aa)

        return props, args_f, args_reg

    def _getValue(self, m, *args):
        """
        Returns the value *J(m)* of the cost function at *m*.
        If the pre-computed values are not supplied `getArguments()` is called.

        :param m: current approximation of the level set function
        :type m: `Data`
        :param args: tuple of values of the parameters, pre-computed values
                     for the forward model and pre-computed values for the
                     regularization
        :rtype: ``float``
        """
        if len(args)==0:
            args=self.getArguments(m)

        props=args[0]
        args_f=args[1]
        args_reg=args[2]

        J = self.regularization.getValue(m, *args_reg)
        self.logger.debug("J_R  (incl. trade-offs) = %e"%J)

        for i in range(self.numModels):
            f, idx=self.forward_models[i]
            args=tuple( [ props[k] for k in idx]  + list( args_f[i] ) )
            J_f = f.getDefect(*args)
            self.logger.debug("J_f[%d] = %e, mu_model[%d] = %e"%(i, J_f, i, self.mu_model[i]))
            J += self.mu_model[i] * J_f

        return J

    def getComponentValues(self, m, *args):
        return self._getComponentValues(m, *args)

    def _getComponentValues(self, m, *args):
        """
        returns the values of the individual cost functions that make up *f(x)*
        using the precalculated values for *x*.

        :param x: a solution approximation
        :type x: x-type
        :rtype: ``list<<float>>``
        """
        if len(args)==0:
            args=self.getArguments(m)

        props=args[0]
        args_f=args[1]
        args_reg=args[2]

        J_reg = self.regularization.getValue(m, *args_reg)
        result = [J_reg]

        for i in range(self.numModels):
            f, idx=self.forward_models[i]
            args=tuple( [ props[k] for k in idx]  + list( args_f[i] ) )
            J_f = f.getValue(*args)
            self.logger.debug("J_f[%d] = %e, mu_model[%d] = %e"%(i, J_f, i, self.mu_model[i]))

            result += [J_f] # self.mu_model[i] * ??

        return result

    def _getGradient(self, m, *args):
        """
        returns the gradient of the cost function at *m*.
        If the pre-computed values are not supplied `getArguments()` is called.

        :param m: current approximation of the level set function
        :type m: `Data`
        :param args: tuple of values of the parameters, pre-computed values
                     for the forward model and pre-computed values for the
                     regularization

        :rtype: `ArithmeticTuple`

        :note: returns (Y^,X) where Y^ is the gradient from regularization plus
               gradients of fwd models. X is the gradient of the regularization
               w.r.t. gradient of m.
        """
        if len(args)==0:
            args = self.getArguments(m)

        props=args[0]
        args_f=args[1]
        args_reg=args[2]

        g_J = self.regularization.getGradient(m, *args_reg)
        p_diffs=[]
        for i in range(self.numMappings):
            mm, idx=self.mappings[i]
            if idx and self.numLevelSets > 1:
                if len(idx)>1:
                    m2=Data(0,(len(idx),),m.getFunctionSpace())
                    for k in range(len(idx)): m2[k]=m[idx[k]]
                    dpdm = mm.getDerivative(m2)
                else:
                    dpdm = mm.getDerivative(m[idx[0]])
            else:
                dpdm = mm.getDerivative(m)
            p_diffs.append(dpdm)

        Y=g_J[0] # Because g_J==(Y,X)  Y_k=dKer/dm_k
        for i in range(self.numModels):
            mu=self.mu_model[i]
            f, idx_f=self.forward_models[i]
            args=tuple( [ props[k] for k in idx_f]  + list( args_f[i] ) )
            Ys = f.getGradient(*args) # this d Jf/d props
            # in this case f depends on one parameter props only but this can
            # still depend on several level set components
            if Ys.getRank() == 0:
                # run through all level sets k prop j is depending on:
                idx_m=self.mappings[idx_f[0]][1]
                # tmp[k] = dJ_f/d_prop * d prop/d m[idx_m[k]]
                tmp=Ys * p_diffs[idx_f[0]] * mu
                if idx_m:
                    if tmp.getRank()== 0:
                        for k in range(len(idx_m)):
                            Y[idx_m[k]]+=tmp # dJ_f /d m[idx_m[k]] = tmp
                    else:
                        for k in range(len(idx_m)):
                            Y[idx_m[k]]+=tmp[k] # dJ_f /d m[idx_m[k]] = tmp[k]
                else:
                    Y+=tmp # dJ_f /d m[idx_m[k]] = tmp
            else:
                s=0
                # run through all props j forward model f is depending on:
                for j in range(len(idx_f)):
                    # run through all level sets k prop j is depending on:
                    idx_m=self.mappings[j][1]
                    if p_diffs[idx_f[j]].getRank() == 0 :
                        if idx_m: # this case is not needed (really?)
                            self.logger.error("something wrong A")
                            # tmp[k] = dJ_f/d_prop[j] * d prop[j]/d m[idx_m[k]]
                            tmp=Ys[s]*p_diffs[idx_f[j]] * mu
                            for k in range(len(idx_m)):
                                Y[idx_m[k]]+=tmp[k] # dJ_f /d m[idx_m[k]] = tmp[k]
                        else:
                            Y+=Ys[s]*p_diffs[idx_f[j]] * mu
                        s+=1
                    elif p_diffs[idx_f[j]].getRank() == 1 :
                        l=p_diffs[idx_f[j]].getShape()[0]
                        # tmp[k]=sum_j dJ_f/d_prop[j] * d prop[j]/d m[idx_m[k]]
                        tmp=inner(Ys[s:s+l], p_diffs[idx_f[j]]) * mu
                        if idx_m:
                            for k in range(len(idx_m)):
                                Y[idx_m[k]]+=tmp # dJ_f /d m[idx_m[k]] = tmp[k]
                        else:
                            Y+=tmp
                        s+=l
                    else: # rank 2 case
                        l=p_diffs[idx_f[j]].getShape()[0]
                        Yss=Ys[s:s+l]
                        if idx_m:
                            for k in range(len(idx_m)):
                                # dJ_f /d m[idx_m[k]] = tmp[k]
                                Y[idx_m[k]]+=inner(Yss, p_diffs[idx_f[j]][:,k])
                        else:
                            Y+=inner(Yss, p_diffs[idx_f[j]]) * mu
                        s+=l
        return g_J

    def _getInverseHessianApproximation(self, m, r, *args):
        """
        returns an approximative evaluation *p* of the inverse of the Hessian
        operator of the cost function for a given gradient type *r* at a
        given location *m*: *H(m) p = r*

        :param m: level set approximation where to calculate Hessian inverse
        :type m: `Data`
        :param r: a given gradient
        :type r: `ArithmeticTuple`
        :param args: tuple of values of the parameters, pre-computed values
                     for the forward model and pre-computed values for the
                     regularization
        :rtype: `Data`
        :note: in the current implementation only the regularization term is
               considered in the inverse Hessian approximation.
        """
        m=self.regularization.getInverseHessianApproximation(m, r, *args[2])
        return m

    def updateHessian(self):
        """
        notifies the class that the Hessian operator needs to be updated.
        """
        self.regularization.updateHessian()

    def _getNorm(self, m):
        """
        returns the norm of `m`

        :param m: level set function
        :type m: `Data`
        :rtype: ``float``
        """
        return self.regularization.getNorm(m)


text:
import numpy as np
import scipy.optimize
import sparse_autoencoder
import unused.stacked_autoencoder as stacked_autoencoder

import used.unused.softmax as softmax
from used import load_MNIST

##======================================================================
## STEP 0: Here we provide the relevant parameters values that will
#  allow your sparse autoencoder to get good filters; you do not need to
#  change the parameters below.

input_size = 28 * 28
num_classes = 10
hidden_size_L1 = 200  # Layer 1 Hidden Size
hidden_size_L2 = 200  # Layer 2 Hidden Size
sparsity_param = 0.1  # desired average activation of the hidden units.
lambda_ = 3e-3  # weight decay parameter
beta = 3  # weight of sparsity penalty term

##======================================================================
## STEP 1: Load data from the MNIST database
#
#  This loads our training data from the MNIST database files.

train_images = load_MNIST.load_MNIST_images('data/mnist/train-images-idx3-ubyte')
train_labels = load_MNIST.load_MNIST_labels('data/mnist/train-labels-idx1-ubyte')

##======================================================================
## STEP 2: Train the first sparse autoencoder
#  This trains the first sparse autoencoder on the unlabelled STL training
#  images.
#  If you've correctly implemented sparseAutoencoderCost.m, you don't need
#  to change anything here.

#  Randomly initialize the parameters
sae1_theta = sparse_autoencoder.initialize(hidden_size_L1, input_size)

J = lambda x: sparse_autoencoder.sparse_autoencoder_cost(x, input_size, hidden_size_L1,
                                                         lambda_, sparsity_param,
                                                         beta, train_images)
options_ = {'maxiter': 400, 'disp': True}

result = scipy.optimize.minimize(J, sae1_theta, method='L-BFGS-B', jac=True, options=options_)
sae1_opt_theta = result.x

print(result)

##======================================================================
## STEP 3: Train the second sparse autoencoder
#  This trains the second sparse autoencoder on the first autoencoder
#  featurse.
#  If you've correctly implemented sparseAutoencoderCost.m, you don't need
#  to change anything here.

sae1_features = sparse_autoencoder.sparse_autoencoder(sae1_opt_theta, hidden_size_L1,
                                                      input_size, train_images)

#  Randomly initialize the parameters
sae2_theta = sparse_autoencoder.initialize(hidden_size_L2, hidden_size_L1)

J = lambda x: sparse_autoencoder.sparse_autoencoder_cost(x, hidden_size_L1, hidden_size_L2,
                                                         lambda_, sparsity_param,
                                                         beta, sae1_features)

options_ = {'maxiter': 400, 'disp': True}

result = scipy.optimize.minimize(J, sae2_theta, method='L-BFGS-B', jac=True, options=options_)
sae2_opt_theta = result.x

print(result)

##======================================================================
## STEP 4: Train the softmax classifier
#  This trains the sparse autoencoder on the second autoencoder features.
#  If you've correctly implemented softmaxCost.m, you don't need
#  to change anything here.

sae2_features = sparse_autoencoder.sparse_autoencoder(sae2_opt_theta, hidden_size_L2,
                                                      hidden_size_L1, sae1_features)

options_ = {'maxiter': 400, 'disp': True}

softmax_theta, softmax_input_size, softmax_num_classes = softmax.softmax_train(hidden_size_L2, num_classes,
                                                                               lambda_, sae2_features,
                                                                               train_labels, options_)

##======================================================================
## STEP 5: Finetune softmax model

# Implement the stacked_autoencoder_cost to give the combined cost of the whole model
# then run this cell.

# Initialize the stack using the parameters learned
stack = [dict() for i in range(2)]
stack[0]['w'] = sae1_opt_theta[0:hidden_size_L1 * input_size].reshape(hidden_size_L1, input_size)
stack[0]['b'] = sae1_opt_theta[2 * hidden_size_L1 * input_size:2 * hidden_size_L1 * input_size + hidden_size_L1]
stack[1]['w'] = sae2_opt_theta[0:hidden_size_L1 * hidden_size_L2].reshape(hidden_size_L2, hidden_size_L1)
stack[1]['b'] = sae2_opt_theta[2 * hidden_size_L1 * hidden_size_L2:2 * hidden_size_L1 * hidden_size_L2 + hidden_size_L2]

# Initialize the parameters for the deep model
(stack_params, net_config) = stacked_autoencoder.stack2params(stack)

stacked_autoencoder_theta = np.concatenate((softmax_theta.flatten(), stack_params))

J = lambda x: stacked_autoencoder.stacked_autoencoder_cost(x, input_size, hidden_size_L2,
                                                           num_classes, net_config, lambda_,
                                                           train_images, train_labels)

options_ = {'maxiter': 400, 'disp': True}
result = scipy.optimize.minimize(J, stacked_autoencoder_theta, method='L-BFGS-B', jac=True, options=options_)
stacked_autoencoder_opt_theta = result.x

print(result)

##======================================================================
## STEP 6: Test

test_images = load_MNIST.load_MNIST_images('data/mnist/t10k-images.idx3-ubyte')
test_labels = load_MNIST.load_MNIST_labels('data/mnist/t10k-labels.idx1-ubyte')

# Two auto encoders without fine tuning
pred = stacked_autoencoder.stacked_autoencoder_predict(stacked_autoencoder_theta, input_size, hidden_size_L2,
                                                       num_classes, net_config, test_images)

print("Before fine-tuning accuracy: {0:.2f}%".format(100 * np.sum(pred == test_labels, dtype=np.float64) /
                                                     test_labels.shape[0]))

# Two auto encoders with fine tuning
pred = stacked_autoencoder.stacked_autoencoder_predict(stacked_autoencoder_opt_theta, input_size, hidden_size_L2,
                                                       num_classes, net_config, test_images)

print("After fine-tuning accuracy: {0:.2f}%".format(100 * np.sum(pred == test_labels, dtype=np.float64) /
                                                    test_labels.shape[0]))

max_stars_repo_name: shifvb/GraduationProject

reason: import numpy,import scipy


text:
#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import parl
import paddle
import numpy as np

class CartpoleAgent(parl.Agent):
    """Agent of Cartpole env.

    Args:
        algorithm(parl.Algorithm): algorithm used to solve the problem.

    """

    def __init__(self, algorithm):
        super(CartpoleAgent, self).__init__(algorithm)

    def sample(self, obs):
        """Sample an action when given an observation

        Args:
            obs(np.float32): shape of (obs_dim,)

        Returns:
            act(int): action
        """
        obs = paddle.to_tensor(obs, dtype='float32')
        prob = self.alg.predict(obs)
        prob = prob.numpy()
        act = np.random.choice(len(prob), 1, p=prob)[0]
        return act

    def predict(self, obs):
        """Predict an action when given an observation

        Args:
            obs(np.float32): shape of (obs_dim,)

        Returns:
            act(int): action
        """
        obs = paddle.to_tensor(obs, dtype='float32')
        prob = self.alg.predict(obs)
        act = prob.argmax().numpy()[0]
        return act

    def learn(self, obs, act, reward):
        """Update model with an episode data

        Args:
            obs(np.float32): shape of (batch_size, obs_dim)
            act(np.int32): shape of (batch_size)
            reward(np.float32): shape of (batch_size)

        Returns:
            loss(float)

        """
        act = np.expand_dims(act, axis=-1)
        reward = np.expand_dims(reward, axis=-1)

        obs = paddle.to_tensor(obs, dtype='float32')
        act = paddle.to_tensor(act, dtype='int32')
        reward = paddle.to_tensor(reward, dtype='float32')

        loss = self.alg.learn(obs, act, reward)
        return loss.numpy()[0]


text:
import numpy as np
def binomial_py(strike):
  """
  Binomial option pricing via looping

  Parameters
  ===========
  strike: Float
    Strike price of the European call option
  """
  # LOOP 1 Index Levels
  s = np.zeros((M + 1, M + 1), dtype = np.float64)
  # Index level array
  s[0,0] = S0
  z1 = 0
  for j in xrange(1, M + 1, 1):
    z1 = z1 + 1
    for i in xrange(z1 + 1):
      S[i, j] = S[0,0] * (u**j) * (d ** (i * 2))
  # Loop 2 - Inner Values
  iv = np.zeros((M+1, M+1), dtype = np.float64)
    # inner value array
  z2 = 0
  for j in xrange(0, M + 1, 1):
    for i in xrange(z2 + 1):
      iv[i, j] = max(S[i, j] - strike, 0)
    z2 = z2 + 1
  # Loop 3 = Valuation
  pv  = np.zeros((M + 1, M + 1), dtype = np.float64)
  # present value array
  pv[:, M] = iv[:, M] #initialize last time point
  z3 = M + 1
  for j in xrange(M - 1, -1, -1):
    z3 = z3 - 1
    for i in xrange(z3):
      pv[i,j] = (q * pv[i, j + 1] + (1 - q) * pv[i + 1, j+1]) * df
  return pv[0,0]

max_stars_repo_name: christopherdurr/Finance-for-Python

max_issues_repo_name: christopherdurr/Finance-for-Python

max_forks_repo_name: christopherdurr/Finance-for-Python


text:
__author__ = 'YanDuarte'

import pandas
import numpy
import statistics
import seaborn
import matplotlib.pyplot as plt
import scipy

# Import the data set to memory
data = pandas.read_csv("separatedData.csv", low_memory = False)

# Change data type among variables to numeric
data["breastCancer100th"] = data["breastCancer100th"].convert_objects(convert_numeric=True)
data["meanSugarPerson"]   = data["meanSugarPerson"].convert_objects(convert_numeric=True)

# Create a subData with only the variables breastCancer100th, meanSugarPerson, meanFoodPerson, meanCholesterol
sub1 = data[['breastCancer100th','meanSugarPerson']]

# Bivariate Scatterplot Q->Q -  Incidence of breast cancer versus sugar consumption
scat1 = seaborn.regplot(x="meanSugarPerson", y="breastCancer100th", fit_reg=True, data=sub1)
plt.xlabel('Mean of the sugar consumption between 1961 and 2002.')
plt.ylabel('Incidence of breast cancer in 100,000 female residents during the 2002 year.')
plt.title('Scatterplot for the association between the incidence of breast cancer and the sugar consumption.')
plt.show()

print ('association between meanSugarPerson and breastCancer100th')
print (scipy.stats.pearsonr(sub1['meanSugarPerson'], sub1['breastCancer100th']))

print('\n r²=')
print(scipy.stats.pearsonr(sub1['meanSugarPerson'], sub1['breastCancer100th'])[0]*scipy.stats.pearsonr(sub1['meanSugarPerson'], sub1['breastCancer100th'])[0])


text:
import pandas as pd
from sklearn.ensemble import RandomForestRegressor
import matplotlib.pyplot as plt
import numpy as np
import forestci
import featurize_monomers
import scipy.stats as stat

seed = 59
feature_strings = featurize_monomers.get_feature_strings()

# Read in the featurized dataset
df = featurize_monomers.get_featurized_dataset("featurized_4-block-polymer_data_SMILES.csv")

# Split monomers into training and testing
# Do sequential learning on testing data
test_i = np.random.choice(list(range(len(df))), int(0.3 * len(df)), replace=False)
test_df = df.iloc[test_i]

# Sort test monomers by band gap * dielectric constant
test_df["Combined"] = test_df[target1] * test_df[target2]
test_df.sort_values("Combined", ascending=False, inplace=True)
test_df.reset_index(inplace=True)

# Store indices of test monomers to track sequential learning performance
test_df["index"] = test_df.index
train_df = df.iloc[[i for i in range(len(df)) if i not in test_i]]

# Get an array of the featurized values
X_test = test_df[feature_strings].values
X_train = train_df[feature_strings].values

y_train = train_df[["HSE Band Gap (eV)", "Total Dielectric Constant"]].values
y_test = test_df[["index", "HSE Band Gap (eV)", "Total Dielectric Constant"]].values

i_list = []
for j in range(len(y_test)):
    print()
    print("Sequential Learning Run #" + str(j))
    print()
    # Create and train models
    model1 = RandomForestRegressor(random_state=seed) # create model
    model1.fit(X_train, y_train[:, 0]) # fit (train) model
    model2 = RandomForestRegressor(random_state=seed) # create model
    model2.fit(X_train, y_train[:, 1]) # fit (train) model

    # Make predictions
    y1_pred = model1.predict(X_test)
    y1_var = forestci.random_forest_error(model1, X_train, X_test)
    y1_std = np.sqrt(sum(y1_var) / len(y1_var)) # Get standard deviation

    y2_pred = model2.predict(X_test)
    y2_var = forestci.random_forest_error(model2, X_train, X_test)
    y2_std = np.sqrt(sum(y2_var) / len(y2_var)) # Get standard deviation

    # get probability of meeting design goals; get Z-scores: (x - mu) / sigma
    Z_bandgap = (5 - y1_pred) / np.sqrt(y1_var)
    Z_dielectric = (5 - y2_pred) / np.sqrt(y2_var)

    # get p-values: 1 - area up to x
    p_bandgap = 1 - stat.norm.cdf(Z_bandgap)
    p_dielectric = 1 - stat.norm.cdf(Z_dielectric)
    probs = p_dielectric * p_bandgap  # joint probability

    # Choose the monomer with the highest probability of meeting our design goals
    besti = int(y_test[np.argmax(probs), 0])
    i_list.append(besti) # Store the rank of the monomer

    # Add the actual DFT values of the monomer to the training data
    X_train = np.append(X_train, X_test[np.abs(y_test[:, 0] - besti) < 1e-1, :].reshape(1, len(feature_strings)), axis=0)
    y_train = np.append(y_train, y_test[np.abs(y_test[:, 0] - besti) < 1e-1, 1:].reshape(1, 2), axis=0)

    # Remove the selected monomer from the testing data
    X_test = X_test[np.abs(y_test[:, 0] - besti) > 1e-1]
    y_test = y_test[np.abs(y_test[:, 0] - besti) > 1e-1]

# Plot the rank of the selected monomer vs sequential learning run number
# Should be linear with slope 1, i.e. best monomer chosen first, worst monomer chosen last
plt.figure()
plt.plot([0, len(i_list)], [0, len(i_list)], color='g', zorder=0)
plt.scatter(list(range(len(i_list))), i_list, color='purple', zorder=100)
plt.legend(['Ideal', 'Actual'])
plt.xlabel("Sequential Learning Run", fontsize=14)
plt.ylabel("Selected Candidate Monomer Rank", fontsize=14)
plt.show()


text:
'''
    Author: David Garcia Fernandez
    Date: 29/6/2019
'''
import matplotlib.pyplot as plt
import numpy as np

final1 = []
final2 = []
for c in range(256):
    for d in range(256):
        final1.append(c)
        final2.append(d)

C = np.array([[255, int(final1[i]), int(final2[i])] for i in range(len(final1))])

fig, ax = plt.subplots(1, figsize=(10, 6))
fig.suptitle('RGB Colors, R = 255', y=0.96, size=16)

ax.scatter(final1, final2, c=C/255.0)
ax.set_xlabel("G", color="#00FF00", size=14)
ax.set_ylabel("B", color="#0000FF", size=14)
plt.show()

max_stars_repo_name: DavidGarciaFer/blog-code

max_issues_repo_name: DavidGarciaFer/blog-code

max_forks_repo_name: DavidGarciaFer/blog-code


text:
#!/usr/bin/python3
# -*- coding: utf-8 -*-

"""Where content is selected"""

__author__ = 'Daniel Campos, Sicong Huang, Hayley Luke, Simola Nayak, Shunjie Wang  '
__email__ = 'dacampos@uw.edu,  huangs33@uw.edu, shunjiew@uw.edu, simnayak@uw.edu, jhluke@uw.edu'

import data_input

import math
import spacy
from scipy import spatial
from sentence_transformers import SentenceTransformer
import numpy as np

def normalize(text_input, nlp,  method='Default'):
    if method == 'Transformer':
        parse = nlp.encode([text_input])[0]
    else:
        parse = nlp(text_input)
        if method == "Noun":
            parse = nlp(' '.join([str(t) for t in parse if t.pos_ in ['NOUN', 'PROPN']]))
        elif method == 'NoStop':
            parse = nlp(' '.join([str(t) for t in parse if not t.is_stop]))
        else:
            parse = 0
    return parse

def process_sentences(nlp, sentences, headline, method):
    headline_nlp = normalize(headline, nlp,  method)
    for sentence in sentences:
        sentence.set_text_parse(normalize(sentence.text, nlp,  method))
        sentence.set_headline_parse(headline_nlp)
    return sentences

def sentence_title_similarity(sentence, method = 'Default'):
    """
    Given a sentence compute its similarity with the topic title. Default method is word vector based
    ARGS: sentence(Sentence Class), method(string)
    Returns: float similarity
    """
    if method == 'Transformer':
        result = 1 - spatial.distance.cosine(sentence.text_nlp, sentence.headline_nlp)
        return result
    else:
        return sentence.text_nlp.similarity(sentence.headline_nlp)

def sentence_similarity(a, b, method = 'Default'):
    """
    Given two sentences produces their cosine similarity. Default is vector similarity
    ARGS: a(Sentence Class), b(Sentence Class), method(string)
    Returns: float similarity
    """
    if method == 'Transformer':
        result = 1 - spatial.distance.cosine(a.text_nlp, b.text_nlp)
        return result
    else:
        return a.text_nlp.similarity(b.text_nlp)

def get_sentences(docs, min_words):
    """
    Given a set of documents we extract all sentences that pass a minimum word threshold
    ARGS: docs(list of Documents), min_words(int)
    Returns: sentences(list of Sentences)
    """
    sentences = []
    [sentences.extend(doc.get_filtered_sentences(min_words)) for doc in docs]
    return sentences

def build_topic_bias(sentences, method):
    """
    Given a list of sentence produce a vector representing the similarity between a topic title(question) and candidate sentences.
    ARGS: sentences(list), method(string)
    Returns: list of distance of each sentence to title(query)
    """
    similarity = []
    for sentence in sentences:
        similarity.append(sentence_title_similarity(sentence, method))
    similarity_sum = np.sum(similarity)
    if similarity_sum == 0:
        similarity_sum = 1
    return np.array(similarity)/ similarity_sum #normalize

def build_matrix(similarity_matrix, topic_bias, dampening):
    """
    Given a sentence similarity matrix, a topic bias vector and a dampening factor produce a unified matrix representation.
    ARGS: similarity_matrix(2d matrix), topic_bias(1d vector), dampening(float)
    Returns: unified matrix(1d)
    """
    return (dampening * (topic_bias)) + ((1-dampening) * (similarity_matrix))

def build_similarity_matrix(sentences, threshold, method):
    """
    Given sentences we Builds and returns a 2D numpy matrix of inter-sentential cosine similarity.
    ARGS: sentences(list), threshold(float), method(str) defining cosine similarity method
    Returns: 2d matrix
    """
    sentence_count = len(sentences)
    similarities = np.zeros((sentence_count, sentence_count))
    for i in range(sentence_count):
        for j in range(i, sentence_count):
            tmp = 1 #default similarity
            if i != j:
                tmp = sentence_similarity(sentences[i], sentences[j], method)
                if tmp > threshold:
                    similarities[i][j] = tmp
                    similarities[j][i] = tmp
    sums = similarities.sum(axis = 1, keepdims=True)
    return similarities/sums #normalize

def get_lex_rank(sentences, matrix, epsilon):
    """
    a matrix and an epsilon value representing the lexical rank of sentences for summaries we use the power method to find lexrank values.
    ARGS: matrix(2d) representing lexrank method
    Returns:  probabilities(1d vector)
    """
    sentence_count = len(sentences)
    probabilities = np.ones(sentence_count)/sentence_count # set even sampling probs
    diff = 1
    while diff > epsilon:
        tmp = np.dot(matrix, probabilities)
        diff = np.linalg.norm(np.subtract(tmp,probabilities))
        probabilities = tmp
    return probabilities

def is_not_too_similar(candidate_sentence, already_selected_sentences, method, similarity_threshold):
    """
    Given a candidate sentence compare to all other already selected sentence and keep only those that aren't within 0.6 of another sentence
    ARGS: candidate_sentence(Sentence), already_selected_sentences(list), method(str)
    Returns: Bool
    """
    similarities = []
    for sentence in already_selected_sentences:
        #print("Sentence 1:{}\nSentence2:{}\nSimilarity:{}".format(candidate_sentence.text, sentence.text,sentence_similarity(candidate_sentence, sentence, method)))
        if sentence_similarity(candidate_sentence, sentence, method) >= similarity_threshold:
            return False
    return True

def get_lex_rank_sorted_sentences(lex_rank_scores):
    """
    Given a list of lex rank scores produce an list representing sentence order sorted by lexrank value
    ARGS: lex_rank_scores(list)
    Returns: lex_rank_sorted_keys(list)
    """
    lex_rank_index_to_score = {}
    for i in range(len(lex_rank_scores)):
        lex_rank_index_to_score[i] = lex_rank_scores[i]
    return list({k: v for k, v in sorted(lex_rank_index_to_score.items(), key=lambda item: item[1], reverse=True)}.keys())

def select_sentences(sentences, sentence_ids_sorted_by_lex_rank, method, similarity_threshold):
    """
    Given a set of sentences and a sorted index via lex rank produce candidate summary by greedily looping over candidates and removing any that make summary > 100 words or are similar to other already selected sentences
    Takes a list of sentences sorted by LexRank value (descending) and selects the sentences to add to the summary greedily based on LexRank value
    while excluding sentences with cosine similarity > 0.6to any sentence already in the summary.
    ARGS: sentences(List of Sentence obj), sentence_ids_sorted_by_lex_rank(list) an ordered list of indexes for sentences based on lexrank score, method(str) setting the cosine comparison method
    Returns: candidate sentences for summary
    """
    current_summary_size = 0
    selected_sentences = []
    for i in sentence_ids_sorted_by_lex_rank:
        cur_sentence_len = len(sentences[i].text.split())
        if cur_sentence_len + current_summary_size < 100:
            if is_not_too_similar(sentences[i], selected_sentences, method, similarity_threshold):
                current_summary_size += cur_sentence_len
                selected_sentences.append(sentences[i])
    return selected_sentences

def select_content(nlp,topics, word_vectors, method, dampening, threshold, epsilon, min_words, similarity_threshold):
    """
    Given a bunch of topics method iterates and creates summaries of <= 100 words using full sentences
    Method uses Biased LExRank Similarity Graph algorithm.
    ARGS: topics, a dampening factor, a inter sentence threshold, an epsilon, min_words
    Returns:
    """

    if method == 'Transformer':
        print('loading Transformers')
        nlp = SentenceTransformer('bert-base-nli-stsb-mean-tokens')
    else:
        print("loading spacy")
        nlp = spacy.load(word_vectors)
    summaries = {}
    idx = 1
    topics_len = len(topics)
    for topic in topics:
        #Get Sentences, process sentences, build similarity matrix, sentence title bias, build markov Matrix, Calculate lexrank, sort sentences by score
        sentences = get_sentences(topic.documents, min_words)
        process_sentences(nlp, sentences, topic.title, method)
        similarity_matrix = build_similarity_matrix(sentences, threshold, method)
        topic_bias = build_topic_bias(sentences, method)
        matrix = build_matrix(similarity_matrix, topic_bias, dampening)
        lex_rank_scores = get_lex_rank(sentences, matrix.T, epsilon) # we trampose matrix for easy math
        sentence_ids_sorted_by_lex_rank = get_lex_rank_sorted_sentences(lex_rank_scores)
        summaries[topic.id] = select_sentences(sentences , sentence_ids_sorted_by_lex_rank, method, similarity_threshold)
        print("Completed {} of {} total topics".format(idx,topics_len))
        idx += 1
    return summaries

reason: import numpy,from scipy


text:
"""
Copyright 2018 Nadheesh Jihan

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
"""
import pymc3 as pm
from matplotlib import pyplot as plt

from bayesian.model_api import Model, _adding_intecept
from data.data_generation import generate_data
from visualizing.plot_kde import plot_kde
from visualizing.predictive_uncertaity import plot_predictive_uncertainty

# HOW TO USE THE MODEL
if __name__ == '__main__':
    n = 500
    d = 5

    # generating data example
    train_X, test_X, train_y, test_y = generate_data(n, d, 0.2)
    train_X, test_X = _adding_intecept(train_X), _adding_intecept(test_X)

    # using model example
    model = Model(n_draws=1000, init_model=None)
    model.fit(train_X, train_y)
    pred, err = model.predict(test_X, with_error=True)
    pm.traceplot(model.trace)

    # plotting uncertainty plots example
    plot_predictive_uncertainty(test_X, test_y, pred, err)

    # plotting kde for given samples example
    fig, ax = plt.subplots()
    coef = model.trace['w'].T
    for w in coef:
        plot_kde(w, ax=ax)

    # display the plots
    plt.show()

max_stars_repo_name: nadheesh/py-scripts

max_issues_repo_name: nadheesh/py-scripts

max_forks_repo_name: nadheesh/py-scripts

reason: import pymc3


text:
import os
import logging
import pandas as pd
import joblib
import scipy.sparse as sparse
import numpy as np

def save_matrix(df, matrix, out_path):
    id_matrix = sparse.csr_matrix(df.id.astype(np.int64)).T
    label_matrix = sparse.csr_matrix(df.label.astype(np.int64)).T

    result = sparse.hstack([id_matrix, label_matrix, matrix], format="csr")

    msg = f"The output matrix {out_path} of size {result.shape} and datatype: {result.dtype}"
    logging.info(msg)

    joblib.dump(result, out_path)


text:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import pandas as pd
import tensorflow as tf
import model_bias_analysis as mba

class ModelBiasAnalysisTest(tf.test.TestCase):
    def add_examples(self, data, model_scores, label, subgroup):
        num_comments_added = len(model_scores)
        data['model_score'].extend(model_scores)
        data['label'].extend([label for a in range(num_comments_added)])
        data['subgroup'].extend([subgroup for a in range(num_comments_added)])

    def test_squared_diff_integral(self):
        x = np.linspace(0.0, 1.0, num = 100)
        y = [1]*len(x)
        result = mba.squared_diff_integral(y, x)
        self.assertAlmostEquals(result, 0.333, places = 2)

    def test_average_squared_equality_gap_no_bias(self):
        no_bias_data = {'model_score': [], 'label': [],
                        'subgroup': []}
        low_model_scores = [0.1, 0.2, 0.22, 0.24, 0.26, 0.28, 0.3, 0.4]
        high_model_scores = [0.7, 0.8, 0.82, 0.84, 0.86, 0.88, 0.9, 1.0]
        self.add_examples(no_bias_data, low_model_scores, 0, False)
        self.add_examples(no_bias_data, low_model_scores, 0, True)
        self.add_examples(no_bias_data, high_model_scores, 1, False)
        self.add_examples(no_bias_data, high_model_scores, 1, True)
        no_bias_df = pd.DataFrame(no_bias_data)

        pos_aseg, neg_aseg = mba.average_squared_equality_gap(no_bias_df,
                                                              'subgroup',
                                                              'label',
                                                              'model_score')
        self.assertAlmostEquals(pos_aseg, 0.0, places = 1)
        self.assertAlmostEquals(neg_aseg, 0.0, places = 1)

    def test_average_squared_equality_gap_small_bias(self):
        no_bias_data = {'model_score': [], 'label': [],
                        'subgroup': []}
        low_model_scores_1 = [0.1, 0.12, 0.14, 0.15, 0.16, 0.18, 0.2]
        low_model_scores_2 = [x + 0.11 for x in low_model_scores_1]
        high_model_scores_1 = [0.7, 0.72, 0.74, 0.75, 0.76, 0.78, 0.8]
        high_model_scores_2 = [x + 0.11 for x in high_model_scores_1]
        self.add_examples(no_bias_data, low_model_scores_1, 0, False)
        self.add_examples(no_bias_data, low_model_scores_2, 0, True)
        self.add_examples(no_bias_data, high_model_scores_1, 1, False)
        self.add_examples(no_bias_data, high_model_scores_2, 1, True)
        no_bias_df = pd.DataFrame(no_bias_data)

        pos_aseg, neg_aseg = mba.average_squared_equality_gap(no_bias_df,
                                                              'subgroup',
                                                              'label',
                                                              'model_score')
        self.assertAlmostEquals(pos_aseg, 0.33, places = 1)
        self.assertAlmostEquals(neg_aseg, 0.33, places = 1)

if __name__ == "__main__":
  tf.test.main()

max_stars_repo_name: ianchou/unintended-ml-bias-analysis

max_issues_repo_name: ianchou/unintended-ml-bias-analysis

max_forks_repo_name: ianchou/unintended-ml-bias-analysis


text:
"""
aep_csm_assembly.py

Created by NWTC Systems Engineering Sub-Task on 2012-08-01.
Copyright (c) NREL. All rights reserved.
"""

import numpy as np

from openmdao.main.api import Component, Assembly, set_as_top, VariableTree
from openmdao.main.datatypes.api import Int, Bool, Float, Array, VarTree, Enum

from fusedwind.plant_flow.asym import BaseAEPModel
from fusedwind.interface import implement_base

from CSMDrivetrain import CSMDrivetrain
from aero_csm_component import aero_csm_component
from aep_csm_component import aep_csm_component

@implement_base(BaseAEPModel)
class aep_csm_assembly(Assembly):

    # Variables
    machine_rating = Float(units = 'kW', iotype='in', desc= 'rated machine power in kW')
    max_tip_speed = Float(units = 'm/s', iotype='in', desc= 'maximum allowable tip speed for the rotor')
    rotor_diameter = Float(units = 'm', iotype='in', desc= 'rotor diameter of the machine')
    max_power_coefficient = Float(iotype='in', desc= 'maximum power coefficient of rotor for operation in region 2')
    opt_tsr = Float(iotype='in', desc= 'optimum tip speed ratio for operation in region 2')
    cut_in_wind_speed = Float(units = 'm/s', iotype='in', desc= 'cut in wind speed for the wind turbine')
    cut_out_wind_speed = Float(units = 'm/s', iotype='in', desc= 'cut out wind speed for the wind turbine')
    hub_height = Float(units = 'm', iotype='in', desc= 'hub height of wind turbine above ground / sea level')
    altitude = Float(units = 'm', iotype='in', desc= 'altitude of wind plant')
    air_density = Float(0.0, units = 'kg / (m * m * m)', iotype='in', desc= 'air density at wind plant site')  # default air density value is 0.0 - forces aero csm to calculate air density in model
    drivetrain_design = Enum('geared', ('geared', 'single_stage', 'multi_drive', 'pm_direct_drive'), iotype='in')
    shear_exponent = Float(iotype='in', desc= 'shear exponent for wind plant') #TODO - could use wind model here
    wind_speed_50m = Float(iotype='in', units = 'm/s', desc='mean annual wind speed at 50 m height')
    weibull_k= Float(iotype='in', desc = 'weibull shape factor for annual wind speed distribution')
    soiling_losses = Float(iotype='in', desc = 'energy losses due to blade soiling for the wind plant - average across turbines')
    array_losses = Float(iotype='in', desc = 'energy losses due to turbine interactions - across entire plant')
    availability = Float(iotype='in', desc = 'average annual availbility of wind turbines at plant')
    turbine_number = Int(iotype='in', desc = 'total number of wind turbines at the plant')
    thrust_coefficient = Float(iotype='in', desc='thrust coefficient at rated power')
    max_efficiency = Float(iotype='in', desc = 'maximum efficiency of rotor and drivetrain - at rated power') # TODO: should come from drivetrain

    # Outputs
    rated_wind_speed = Float(11.506, units = 'm / s', iotype='out', desc='wind speed for rated power')
    rated_rotor_speed = Float(12.126, units = 'rpm', iotype='out', desc = 'rotor speed at rated power')
    rotor_thrust = Float(iotype='out', units='N', desc='maximum thrust from rotor')
    rotor_torque = Float(iotype='out', units='N * m', desc = 'torque from rotor at rated power')
    power_curve = Array(np.array([[4.0,80.0],[25.0, 5000.0]]), iotype='out', desc = 'power curve for a particular rotor')
    #max_efficiency = Float(0.902, iotype='out', desc = 'maximum efficiency of rotor and drivetrain - at rated power')  #TODO: should come from drivetrain
    gross_aep = Float(0.0, iotype='out', desc='Gross Annual Energy Production before availability and loss impacts', unit='kWh')
    net_aep = Float(0.0, units= 'kW * h', iotype='out', desc='Annual energy production in kWh')  # use PhysicalUnits to set units='kWh'
    capacity_factor = Float(iotype='out', desc='plant capacity factor')

    def configure(self):
        ''' configures assembly by adding components, creating the workflow, and connecting the component i/o within the workflow '''

        self.add('drive', CSMDrivetrain())
        self.add('aero',aero_csm_component())
        self.add('aep',aep_csm_component())

        self.driver.workflow.add(['drive', 'aero', 'aep'])

        # connect inputs to component inputs
        self.connect('machine_rating', ['aero.machine_rating', 'drive.ratedPower','aep.machine_rating'])
        self.connect('hub_height', ['aero.hub_height', 'aep.hub_height'])

        # connect i/o between components
        #self.connect('drive.drivetrain','aero.drivetrain')
        self.connect('aero.power_curve','drive.aeroPower')
        self.connect('drive.power',['aep.power_curve'])
        self.connect('aero.wind_curve', 'aep.wind_curve')
        self.connect('aep.power_array','power_curve')

        # turbine
        self.connect('max_tip_speed','aero.max_tip_speed')
        self.connect('rotor_diameter','aero.rotor_diameter')
        self.connect('max_power_coefficient','aero.max_power_coefficient')
        self.connect('opt_tsr','aero.opt_tsr')
        self.connect('cut_in_wind_speed','aero.cut_in_wind_speed')
        self.connect('cut_out_wind_speed','aero.cut_out_wind_speed')
        self.connect('drivetrain_design','drive.drivetrainType')
        self.connect('thrust_coefficient','aero.thrust_coefficient')
        self.connect('max_efficiency', 'aero.max_efficiency')
        # plant
        self.connect('shear_exponent','aep.shear_exponent')
        self.connect('weibull_k','aep.weibull_k')
        self.connect('wind_speed_50m', 'aep.wind_speed_50m')
        self.connect('soiling_losses','aep.soiling_losses')
        self.connect('array_losses','aep.array_losses')
        self.connect('availability', 'aep.availability')
        self.connect('turbine_number','aep.turbine_number')
        self.connect('altitude','aero.altitude')
        self.connect('air_density','aero.air_density')

        # connect outputs
        self.connect('aero.rated_rotor_speed','rated_rotor_speed')
        self.connect('aero.rated_wind_speed', 'rated_wind_speed')
        self.connect('aero.rotor_thrust','rotor_thrust')
        self.connect('aero.rotor_torque','rotor_torque')
        self.connect('aep.gross_aep', 'gross_aep')
        self.connect('aep.net_aep', 'net_aep')
        self.connect('aep.capacity_factor', 'capacity_factor')
        #self.create_passthrough('aep.aep_per_turbine')

def example():

    aepA = aep_csm_assembly()

    aepA.machine_rating = 5000.0 # Float(units = 'kW', iotype='in', desc= 'rated machine power in kW')
    aepA.rotor_diameter = 126.0 # Float(units = 'm', iotype='in', desc= 'rotor diameter of the machine')
    aepA.max_tip_speed = 80.0 # Float(units = 'm/s', iotype='in', desc= 'maximum allowable tip speed for the rotor')
    aepA.drivetrain_design = 'geared' # Enum('geared', ('geared', 'single_stage', 'multi_drive', 'pm_direct_drive'), iotype='in')
    aepA.altitude = 0.0 # Float(0.0, units = 'm', iotype='in', desc= 'altitude of wind plant')
    aepA.turbine_number = 100 # Int(100, iotype='in', desc = 'total number of wind turbines at the plant')
    aepA.hub_height = 90.0 # Float(units = 'm', iotype='in', desc='hub height of wind turbine above ground / sea level')s
    aepA.max_power_coefficient = 0.488 #Float(0.488, iotype='in', desc= 'maximum power coefficient of rotor for operation in region 2')
    aepA.opt_tsr = 7.525 #Float(7.525, iotype='in', desc= 'optimum tip speed ratio for operation in region 2')
    aepA.cut_in_wind_speed = 3.0 #Float(3.0, units = 'm/s', iotype='in', desc= 'cut in wind speed for the wind turbine')
    aepA.cut_out_wind_speed = 25.0 #Float(25.0, units = 'm/s', iotype='in', desc= 'cut out wind speed for the wind turbine')
    aepA.hub_height = 90.0 #Float(90.0, units = 'm', iotype='in', desc= 'hub height of wind turbine above ground / sea level')
    #aepA.air_density = Float(0.0, units = 'kg / (m * m * m)', iotype='in', desc= 'air density at wind plant site')  # default air density value is 0.0 - forces aero csm to calculate air density in model
    aepA.shear_exponent = 0.1 #Float(0.1, iotype='in', desc= 'shear exponent for wind plant') #TODO - could use wind model here
    aepA.wind_speed_50m = 8.02 #Float(8.35, units = 'm/s', iotype='in', desc='mean annual wind speed at 50 m height')
    aepA.weibull_k= 2.15 #Float(2.1, iotype='in', desc = 'weibull shape factor for annual wind speed distribution')
    aepA.soiling_losses = 0.0 #Float(0.0, iotype='in', desc = 'energy losses due to blade soiling for the wind plant - average across turbines')
    aepA.array_losses = 0.10 #Float(0.06, iotype='in', desc = 'energy losses due to turbine interactions - across entire plant')
    aepA.availability = 0.941 #Float(0.94287630736, iotype='in', desc = 'average annual availbility of wind turbines at plant')
    aepA.thrust_coefficient = 0.50 #Float(0.50, iotype='in', desc='thrust coefficient at rated power')
    aepA.max_efficiency = 0.902

    aepA.run()

    print "Annual energy production for an offshore wind plant with 100 NREL 5 MW reference turbines."
    print "AEP gross output (before losses): {0:.1f} kWh".format(aepA.gross_aep)
    print "AEP net output (after losses): {0:.1f} kWh".format(aepA.net_aep)
    print "Rated rotor speed: {0:.2f} rpm".format(aepA.rated_rotor_speed)
    print "Rated wind speed: {0:.2f} m/s".format(aepA.rated_wind_speed)

if __name__=="__main__":

    example()


text:
#  -*- coding: utf-8 -*-
# *****************************************************************************
# ufit, a universal scattering fitting suite
#
# Copyright (c) 2013-2019, Georg Brandl and contributors.  All rights reserved.
# Licensed under a 2-clause BSD license, see LICENSE.
# *****************************************************************************

"""Various dialogs."""

from numpy import array, savetxt, nan

from ufit.qt import pyqtSlot, QDialog, QListWidgetItem

from ufit.gui.common import loadUi
from ufit.data.dataset import Dataset

class ParamExportDialog(QDialog):
    def __init__(self, parent, items):
        QDialog.__init__(self, parent)
        loadUi(self, 'paramselect.ui')
        self.availList.populate(items, intersect=False)
        self.items = items

    def on_availList_itemDoubleClicked(self, item):
        QListWidgetItem(item.text(), self.selectList, item.type())

    @pyqtSlot()
    def on_addBtn_clicked(self):
        item = self.availList.currentItem()
        if not item:
            return
        QListWidgetItem(item.text(), self.selectList, item.type())

    @pyqtSlot()
    def on_removeBtn_clicked(self):
        item = self.selectList.currentItem()
        if not item:
            return
        self.selectList.takeItem(self.selectList.row(item))

    def do_export(self, filename):
        data = []
        colnames = [self.selectList.item(i).text().strip()
                    for i in range(self.selectList.count())]
        coltypes = [self.selectList.item(i).type()
                    for i in range(self.selectList.count())]
        errors = self.errorBox.isChecked()
        for item in self.items:
            drow = []
            for coltype, name in zip(coltypes, colnames):
                if coltype == 1:   # parameter
                    if name in item.model.paramdict:
                        drow.append(item.model[name].value)
                        if errors:
                            drow.append(item.model[name].error)
                    else:
                        drow.append(nan)
                        if errors:
                            drow.append(nan)
                else:    # data value
                    if name in item.data.meta:
                        drow.append(item.data.meta[name])
                    else:
                        drow.append(nan)
            data.append(drow)
        data = array(data)
        if self.headerBox.isChecked():
            header = []
            for coltype, name in zip(coltypes, colnames):
                header.append(name)
                if coltype == 1 and self.errorBox.isChecked():
                    header.append(name + '_error')
            header = '\t'.join(header)
        else:
            header = ''
        savetxt(filename, data, header=header)

class ParamSetDialog(QDialog):
    def __init__(self, parent, items):
        QDialog.__init__(self, parent)
        loadUi(self, 'paramset.ui')
        self.new_data = None
        self.items = items
        self.xaxisList.populate(items)
        self.yaxisList.populate(items)
        self._auto_name = ''

    def _gen_name(self):
        xi = self.xaxisList.currentItem()
        yi = self.yaxisList.currentItem()
        if self.nameBox.text() != self._auto_name:
            return
        auto_name = ''
        if yi:
            auto_name += yi.text().strip()
        if xi:
            auto_name += ' vs. ' + xi.text().strip()
        self._auto_name = auto_name
        self.nameBox.setText(auto_name)

    def on_xaxisList_itemSelectionChanged(self):
        self._gen_name()

    def on_yaxisList_itemSelectionChanged(self):
        self._gen_name()

    def exec_(self):
        res = QDialog.exec_(self)
        if res != QDialog.Accepted:
            return res
        xx, yy, dy = [], [], []
        xp = yp = False
        xv = self.xaxisList.currentItem().text().strip()
        if self.xaxisList.currentItem().type() == 1:
            xp = True
        yv = self.yaxisList.currentItem().text().strip()
        if self.yaxisList.currentItem().type() == 1:
            yp = True

        for item in self.items:
            if xp:
                xx.append(item.model.paramdict[xv].value)
            else:
                xx.append(item.data.meta[xv])
            if yp:
                yy.append(item.model.paramdict[yv].value)
                dy.append(item.model.paramdict[yv].error)
            else:
                yy.append(item.data.meta[yv])
                dy.append(1)
        xx, yy, dy = map(array, [xx, yy, dy])

        self.new_data = Dataset.from_arrays(str(self.nameBox.text()),
                                            xx, yy, dy, xcol=xv, ycol=yv)
        return res

max_stars_repo_name: McStasMcXtrace/ufit

max_issues_repo_name: McStasMcXtrace/ufit

max_forks_repo_name: McStasMcXtrace/ufit

reason: from numpy